Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Throttling Imports #42

Open
wants to merge 3 commits into
base: integration
Choose a base branch
from
Open

Throttling Imports #42

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
34c5c45
add support for importing using entry id"
jonmifsud Jan 15, 2015
3690022
add throttle support
jonmifsud Jan 20, 2015
e42bead
xml importer fixes
jonmifsud Jan 25, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 34 additions & 0 deletions assets/xmlimporter.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,4 +18,38 @@
}).trigger('change.xmlimporter');
});

$(document).on('click','.throttle',function(){
var data = {
"created":0,
"updated":0,
"skipped":0,
"failed":0,
};
data.next = $('.next-page').data('next');
data.nexturi = $('.next-page').data('uri');
throttle(data,0);
});

function throttle(data,maxpage){
$importdetails = $('.xmlimport-details');
$importdetails.data('created', $importdetails.data('created') + data.created);
$importdetails.data('updated', $importdetails.data('updated') + data.updated);
$importdetails.data('skipped', $importdetails.data('skipped') + data.skipped);
$importdetails.data('failed', $importdetails.data('failed') + data.failed);
$importdetails.text( $importdetails.data('created') + ' new entries were created, ' +
$importdetails.data('updated') +' updated, ' +
$importdetails.data('skipped') +' skipped and ' +
$importdetails.data('failed') +' failed'
);

if (data.next != null && (data.next <= maxpage || maxpage == 0)){
$importdetails.prev().text('Currently Importing Page ' + data.next);
$.getJSON( data.nexturi + '&ajax=1' , function( data ) {
throttle(data,maxpage);
});
} else {
$importdetails.prev().text('Import Complete');
}
}

})(jQuery);
232 changes: 230 additions & 2 deletions content/content.importers.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -276,7 +276,13 @@ public function __viewRun() {
$importer_result['updated'],
$importer_result['skipped'],
$importer_result['failed']
))
)), array(
'data-created' => $importer_result['created'],
'data-updated' => $importer_result['updated'],
'data-skipped' => $importer_result['skipped'],
'data-failed' => $importer_result['failed'],
'class' => 'xmlimport-details',
)
));

// Import errors
Expand All @@ -303,9 +309,35 @@ public function __viewRun() {
$importer_result['created'],
$importer_result['updated'],
$importer_result['skipped']
))
)), array(
'data-created' => $importer_result['created'],
'data-updated' => $importer_result['updated'],
'data-skipped' => $importer_result['skipped'],
'data-failed' => $importer_result['failed'],
'class' => 'xmlimport-details',
)
));

$throttle = $importer->getThrottle();
$pagination = $importer->pagination();
if ( !empty($throttle)){
$fieldset->appendChild(Widget::Anchor(
__('Next Page'),
$this->_uri . '/importers/run/' . $this->_context[1] . '/?' . $pagination['variable'] . '=' . $throttle['next-page'],
__('Next Page'),
'button next-page',null,array(
'data-next'=>$throttle['next-page'],
'data-uri' => $this->_uri . '/importers/run/' . $this->_context[1] . '/?' . $pagination['variable'] . '=' . $throttle['next-page']
)
));
$fieldset->appendChild(Widget::Anchor(
__('Throttle'),
'#throttle',
__('Throttle'),
'button throttle'
));
}

}

$this->Form->appendChild($fieldset);
Expand All @@ -323,6 +355,37 @@ public function __viewRun() {
'entries' => $entries
)
);

$throttle = $importer->getThrottle();
$pagination = $importer->pagination();
if ($throttle['ajax']){

$failedEntries = new XMLElement(
'div', null, array('class'=>'failed-entries')
);

if (!empty($failed)){
$fieldset->appendChild(new XMLElement(
'h3', __('Import Errors: Page %d', array(
$throttle['current-page']
))
));

$this->addFailedEntries($failedEntries, $failed);
}

echo json_encode( array(
'created' => $importer_result['created'],
'updated' => $importer_result['updated'],
'skipped' => $importer_result['skipped'],
'failed' => $importer_result['failed'],
'next' => $throttle['next-page'],
'nexturi' => $this->_uri . '/importers/run/' . $this->_context[1] . '/?' . $pagination['variable'] . '=' . $throttle['next-page'],
'errors' => $failedEntries->generate()
));

die;
}
}
}

Expand Down Expand Up @@ -459,6 +522,18 @@ public function __actionEditNormal() {
: 'no'
);

// pagination/throttling settings

// validate has next page xpath
if (!empty($fields['pagination']['next'])){
try {
$this->_driver->validateXPath($fields['pagination']['next'], $fields['namespaces']);
}
catch (Exception $e) {
$this->_errors['pagination']['next'] = $e->getMessage();
}
}

$this->_fields = $fields;

if (!empty($this->_errors)) {
Expand Down Expand Up @@ -669,6 +744,81 @@ public function __viewEdit() {

$this->Form->appendChild($fieldset);

// Pagination --------------------------------------------------


$fieldset = new XMLElement('fieldset');
$fieldset->setAttribute('class', 'settings');
$fieldset->appendChild(new XMLElement('legend', __('Throttling')));

$group = new XMLElement('div');
$group->setAttribute('class', 'two columns');

$label = Widget::Label(__('Pagination Variable <i>Optional</i>'));
$label->setAttribute('class', 'column');
$input = Widget::Input(
'fields[pagination][variable]',
General::sanitize(
isset($this->_fields['pagination']['variable'])
? $this->_fields['pagination']['variable']
: null
)
);
$input->setAttribute('placeholder', 'p');
$label->appendChild($input);

if (isset($this->_errors['variable'])) {
$label = Widget::Error($label, $this->_errors['variable']);
}

$group->appendChild($label);

$label = Widget::Label(__('Start Page <i>Optional</i>'));
$label->setAttribute('class', 'column');
$input = Widget::Input(
'fields[pagination][start]',
General::sanitize(
isset($this->_fields['pagination']['start'])
? $this->_fields['pagination']['start']
: null
)
);
$input->setAttribute('placeholder', '1');
$label->appendChild($input);

$fieldset->appendChild($group);

$group->appendChild($label);
$help = new XMLElement('p');
$help->setAttribute('class', 'help');
$help->setValue(__('The pagination variable has to be included within your Datasource as a url parameter. Not setting a variable will ignore throttling.'));
$fieldset->appendChild($help);

$label = Widget::Label(__('Next Page <i>Optional</i>'));
$input = Widget::Input(
'fields[pagination][next]',
General::sanitize(
isset($this->_fields['pagination']['next'])
? $this->_fields['pagination']['next']
: null
)
);
$input->setAttribute('placeholder', '/data/datasource/pagination/@current-page != /data/datasource/pagination/@total-pages');
$label->appendChild($input);

if (isset($this->_errors['pagination']['next'])) {
$label = Widget::Error($label, $this->_errors['pagination']['next']);
}

$fieldset->appendChild($label);

$help = new XMLElement('p');
$help->setAttribute('class', 'help');
$help->setValue(__('Use an XPath expression to determine if there is a next page (boolean).'));
$fieldset->appendChild($help);

$this->Form->appendChild($fieldset);

// Section ------------------------------------------------------------

$sections = SectionManager::fetch(null, 'ASC', 'name');
Expand Down Expand Up @@ -713,6 +863,84 @@ public function __viewEdit() {

if ($fields === false) continue;

//entry ID markup
$field_id = 'entry-id';
// $field_name = "fields[fields][-1]";
$field_name = "fields[fields][id]"; //not sure that is correct but should avoid conflicts with other fields
$field_data = null;
$template_index = null;
if (isset($this->_fields['fields'])) {
foreach ($this->_fields['fields'] as $i => $temp_data) {
if ($temp_data['field'] != $field_id) continue;
$field_data = $temp_data;
$template_index = $i;
// always force the unique to entry id if exists
$this->_fields['unique-field'] = "entry-id";
}
}
$li = new XMLElement('li',"<h4>Entry ID</h4>",array('class'=>'unique template','data-type'=>'entry-id'));
$li->appendChild(new XMLElement('header', '<h4>Entry ID</h4>'));

$input = Widget::Input("{$field_name}[field]", $field_id, 'hidden');
$li->appendChild($input);

$group = new XMLElement('div');
$group->setAttribute('class', 'two columns');

$label = Widget::Label(__('XPath Expression'));
$label->setAttribute('class', 'column');
$input = Widget::Input(
"{$field_name}[xpath]",
General::sanitize(
( isset($field_data) && isset($field_data['xpath']) )
? $field_data['xpath']
: null
)
);
$label->appendChild($input);
$group->appendChild($label);

$label = Widget::Label(__('PHP Function'));
$label->appendChild(new XMLElement('i', __('Optional')));
$label->setAttribute('class', 'column');
$input = Widget::Input(
"{$field_name}[php]",
General::sanitize(
( isset($field_data) && isset($field_data['php']) )
? $field_data['php']
: null
)
);
$label->appendChild($input);
$group->appendChild($label);

$li->appendChild($group);

$label = Widget::Label();
$label->setAttribute('class', 'meta');

$label->setValue(__('Entry ID is used to determine uniqeness.'));

$li->appendChild($label);
$section_fields->appendChild($li);

if (!is_null($field_data)){
//clone to avoid re-setting all the variables
$newLi = clone $li;
$newLi->setAttribute('class', 'unique');

//an entry ID must be unique - check only when showing in selected view
$input = Widget::Input("fields[unique-field]", $field_id, 'radio');
$input->setAttribute("checked","checked");
$input->setAttribute("style","display:none");
$newLi->appendChild($input);

//append item to the field list
$section_fields->appendChild($newLi);
}

//end entry id

// Templates
foreach ($fields as $index => $field) {
$field_id = $field->get('id');
Expand Down
8 changes: 7 additions & 1 deletion extension.driver.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,7 @@ public function getXMLImporter($name) {

$data = $importer->options();
$data['about'] = $importer->about();
$data['pagination'] = $importer->pagination();

return $data;
}
Expand Down Expand Up @@ -204,7 +205,12 @@ public function setXMLImporter(&$name, &$error, $new) {
var_export($new['source'], true),
var_export($new['timeout'], true),
var_export($new['section'], true),
var_export($new['unique-field'], true)
var_export($new['unique-field'], true),

// pagination
var_export($new['pagination']['variable'], true),
var_export($new['pagination']['start'], true),
var_export($new['pagination']['next'], true)
);

// Write file to disk:
Expand Down
Loading