From e223c7ee620ac166f9baec5dbabed27c0e77f994 Mon Sep 17 00:00:00 2001 From: Shaun VanWeelden Date: Sat, 28 Dec 2019 12:18:31 -0800 Subject: [PATCH 1/2] Add Code Examples --- examples/cancelTasks.js | 82 +++++++++++++++++++++++++++++++++++ examples/createNewTasks.js | 89 ++++++++++++++++++++++++++++++++++++++ examples/getAllTasks.js | 84 +++++++++++++++++++++++++++++++++++ 3 files changed, 255 insertions(+) create mode 100644 examples/cancelTasks.js create mode 100644 examples/createNewTasks.js create mode 100644 examples/getAllTasks.js diff --git a/examples/cancelTasks.js b/examples/cancelTasks.js new file mode 100644 index 0000000..dc89c99 --- /dev/null +++ b/examples/cancelTasks.js @@ -0,0 +1,82 @@ +const scaleapi = require('../lib/scaleapi.js'); // Change to just "scaleapi" for your project +const fs = require('fs'); + +// HOW IT WORKS: +// Given a .csv file or array of task ids you'd like cancelled, +// go through and cancel Tasks for each row. + +// INPUT PARAMETERS: +const API_KEY = 'live_xxxxx'; +const DO_DRY_RUN = true; +const fileName = 'list/of/task/ids_to_cancel.csv' + +const client = scaleapi.ScaleClient(API_KEY); + +(async function () { + // ==================== + // === READ IN ROWS === + // ==================== + + // Read in Task Details from CSV + let rows = getIdsFromCsv(fileName); + + // Alternatively, create just an array of rows to cancel Tasks from + // let rows = [ + // '5d4121900591c138750xxxxx' + // ] + + // ==================== + // === PROCESS ROWS === + // ==================== + + // Process each row as needed + rows = rows.map( + row => row.split(',')[0].trim() + ); + + console.log(`Number of Rows Found: ${rows.length}`); + + // ==================== + // === CANCEL TASKS === + // ==================== + + if (rows.length > 0) { + await Promise.map( + rows, + async row => { + if (DO_DRY_RUN) { + console.log('Would be cancelling Task Id: ' + row); + } else { + await new Promise((resolve, reject) => { + client.cancelTask(row, (err, task) => { + // do something with task + if (err) { + console.error(err); + reject(err); + } else { + console.log(`Task Cancelled: ${task.task_id}`); + resolve(); + } + }); + }); + } + }, + { concurrency: 10 }, + ); + + console.log('Finished Running Script'); + } +}()); + +let readCsv = function(fileName, hasHeader = false) { + const rows = + fs.readFileSync(fileName, { encoding: 'utf8' }) + .split('\n') + .map(s => s.trim()) || []; + + return hasHeader && rows.length > 0 ? rows.splice(1) : rows; +} + +let getIdsFromCsv = function(fileName, hasHeader) { + return readCsv(fileName, hasHeader).filter(s => s.length === 24); +}; \ No newline at end of file diff --git a/examples/createNewTasks.js b/examples/createNewTasks.js new file mode 100644 index 0000000..7c65418 --- /dev/null +++ b/examples/createNewTasks.js @@ -0,0 +1,89 @@ +const scaleapi = require('../lib/scaleapi.js'); // Change to just "scaleapi" for your project +const fs = require('fs'); + +// HOW IT WORKS: +// Given a .csv file or array of information to make tasks from (often attachment urls), +// go through and submit Tasks to Scale for each row. + +// You will want to specify which task creation method you'd like to use, as well as +// any input parameters relating to how you'd like your task completed + +// INPUT PARAMETERS: +const API_KEY = 'live_xxxxx'; +const DO_DRY_RUN = true; +const fileName = 'list/of/attachment_urls_and_other_data.csv' + +const client = scaleapi.ScaleClient(API_KEY); + +(async function () { + // ==================== + // === READ IN ROWS === + // ==================== + + // Read in Task Details from CSV + let rows = readCsv(fileName); + + // Alternatively, create just an array of rows to create Tasks from + // let rows = [ + // 'https://www.scale.com/img/is/awesome.jpg' + // ] + + // ==================== + // === PROCESS ROWS === + // ==================== + + // Process each row as needed + rows = rows.map(row => row.split(',')[0].trim()); + + console.log(`Number of Rows Found: ${rows.length}`); + + // ==================== + // === CREATE TASKS === + // ==================== + + if (rows.length > 0) { + await Promise.map( + rows, + async row => { + if (DO_DRY_RUN) { + console.log('Creating Task for ' + row); + } else { + await new Promise((resolve, reject) => { + client.createAnnotationTask( + { + callback_url: 'http://www.example.com/callback', + project: 'coolest_project_name', + objects_to_annotate: ['person', 'land vehicle'], + with_labels: true, + attachment: row, + attachment_type: 'image', + }, + (err, task) => { + // do something with task + if (err) { + console.error(err); + reject(err); + } else { + console.log(`Task Created: ${task.task_id}`); + resolve(); + } + }, + ); + }); + } + }, + { concurrency: 5 }, + ); + + console.log('Finished Running Script'); + } +}()); + +function readCsv(fileName, hasHeader = false) { + const rows = + fs.readFileSync(fileName, { encoding: 'utf8' }) + .split('\n') + .map(s => s.trim()) || []; + + return hasHeader && rows.length > 0 ? rows.splice(1) : rows; +} \ No newline at end of file diff --git a/examples/getAllTasks.js b/examples/getAllTasks.js new file mode 100644 index 0000000..aeb8d0c --- /dev/null +++ b/examples/getAllTasks.js @@ -0,0 +1,84 @@ +const scaleapi = require('../lib/scaleapi.js'); // Change to just "scaleapi" for your project +const fs = require('fs'); + +// HOW IT WORKS: +// Given a list of search filters ("PARAMS"), it will page through tasks and write +// the output to a JSON file. + + +// INPUT PARAMETERS: +const API_KEY = 'live_xxxxx'; + +const MAX_TASKS_TO_RETURN = 100000; // Get up to the n most recently created tasks + +const OUTPUT_FILE = '/the/place/to/put/it.json'; + +const PARAMS = { // All params optional + type: 'annotation', + status: 'completed', + project: 'cool_project_name', + completed_after: '2019-03-01T00:00:00.000Z' +}; + +(async function () { + + // =============================== + // == MAIN FUNCTION WE CAN CALL == + // =============================== + + // Get list of task objects + let getTasks = async function(client, maxTasksToReturn = 1000000, params = {}) { + // Initialize everything + const maxTasksReturnedPerCall = 100; + let lastPageCount = maxTasksReturnedPerCall; + let output = []; + + // Go through page by page + while (lastPageCount == maxTasksReturnedPerCall && output.length < maxTasksToReturn) { + + // Skip ahead (offset) by the number of tasks already pulled + params.offset = output.length; + + // Support asking for small number of tasks (like 5, 10, 50) + params.limit = maxTasksReturnedPerCall; + if (maxTasksToReturn - output.length < maxTasksReturnedPerCall) { + params.limit = maxTasksToReturn - output.length + } + + try { + // fetch some tasks + let tasks = await new Promise((resolve, reject) => { + client.tasks(params, (err, tasklist) => { + if (err) { + console.error(err); + reject(err); + } else { + resolve(tasklist); + } + }); + }); + + // concat with output + output = output.concat(tasks.docs); + lastPageCount = tasks.docs.length; + + console.log(`Fetched ${output.length} tasks`); + } catch(err) { + console.error(err) + } + } + + console.log(`Finished fetching ${output.length} tasks`); + + return output; + } + + // ============================ + // == CALL OUR MAIN FUNCTION == + // ============================ + + const client = new scaleapi.ScaleClient(API_KEY); + let tasks = await getTasks(client, MAX_TASKS_TO_RETURN, PARAMS); + fs.writeFileSync(OUTPUT_FILE, JSON.stringify(tasks)); +}()); + From a548fa61768d67980b1b46dc88689c34709cd149 Mon Sep 17 00:00:00 2001 From: Shaun VanWeelden Date: Fri, 24 Jan 2020 15:20:26 -0800 Subject: [PATCH 2/2] Changes based on PR Feedback --- examples/cancelTasks.js | 66 +++++++++++++++++--------------------- examples/createNewTasks.js | 12 +++---- examples/getAllTasks.js | 26 ++++++--------- 3 files changed, 45 insertions(+), 59 deletions(-) diff --git a/examples/cancelTasks.js b/examples/cancelTasks.js index dc89c99..c5161b3 100644 --- a/examples/cancelTasks.js +++ b/examples/cancelTasks.js @@ -1,4 +1,4 @@ -const scaleapi = require('../lib/scaleapi.js'); // Change to just "scaleapi" for your project +const scaleapi = require('scaleapi'); // Change to "../lib/scaleapi.js" if you intend to run in this repo const fs = require('fs'); // HOW IT WORKS: @@ -18,7 +18,7 @@ const client = scaleapi.ScaleClient(API_KEY); // ==================== // Read in Task Details from CSV - let rows = getIdsFromCsv(fileName); + let rows = readCsv(fileName); // Alternatively, create just an array of rows to cancel Tasks from // let rows = [ @@ -30,9 +30,7 @@ const client = scaleapi.ScaleClient(API_KEY); // ==================== // Process each row as needed - rows = rows.map( - row => row.split(',')[0].trim() - ); + rows = rows.map(row => row[0]).filter(id => id.length === 24); console.log(`Number of Rows Found: ${rows.length}`); @@ -40,43 +38,37 @@ const client = scaleapi.ScaleClient(API_KEY); // === CANCEL TASKS === // ==================== - if (rows.length > 0) { - await Promise.map( - rows, - async row => { - if (DO_DRY_RUN) { - console.log('Would be cancelling Task Id: ' + row); - } else { - await new Promise((resolve, reject) => { - client.cancelTask(row, (err, task) => { - // do something with task - if (err) { - console.error(err); - reject(err); - } else { - console.log(`Task Cancelled: ${task.task_id}`); - resolve(); - } - }); + await Promise.map( + rows, + async row => { + if (DO_DRY_RUN) { + console.log('Would be cancelling Task Id: ' + row); + } else { + await new Promise((resolve, reject) => { + client.cancelTask(row, (err, task) => { + // do something with task + if (err) { + console.error(err); + reject(err); + } else { + console.log(`Task Cancelled: ${task.task_id}`); + resolve(); + } }); - } - }, - { concurrency: 10 }, - ); + }); + } + }, + { concurrency: 10 }, + ); - console.log('Finished Running Script'); - } + console.log('Finished Running Script'); }()); -let readCsv = function(fileName, hasHeader = false) { +function readCsv(fileName, hasHeader = false) { const rows = fs.readFileSync(fileName, { encoding: 'utf8' }) .split('\n') - .map(s => s.trim()) || []; - - return hasHeader && rows.length > 0 ? rows.splice(1) : rows; -} + .map(r => r.split(",").map(s => s.trim())) || []; -let getIdsFromCsv = function(fileName, hasHeader) { - return readCsv(fileName, hasHeader).filter(s => s.length === 24); -}; \ No newline at end of file + return hasHeader ? rows.splice(1) : rows; +} \ No newline at end of file diff --git a/examples/createNewTasks.js b/examples/createNewTasks.js index 7c65418..f80e4b8 100644 --- a/examples/createNewTasks.js +++ b/examples/createNewTasks.js @@ -1,4 +1,4 @@ -const scaleapi = require('../lib/scaleapi.js'); // Change to just "scaleapi" for your project +const scaleapi = require('scaleapi'); // Change to "../lib/scaleapi.js" if you intend to run in this repo const fs = require('fs'); // HOW IT WORKS: @@ -10,7 +10,7 @@ const fs = require('fs'); // INPUT PARAMETERS: const API_KEY = 'live_xxxxx'; -const DO_DRY_RUN = true; +const DRY_RUN = true; const fileName = 'list/of/attachment_urls_and_other_data.csv' const client = scaleapi.ScaleClient(API_KEY); @@ -32,8 +32,8 @@ const client = scaleapi.ScaleClient(API_KEY); // === PROCESS ROWS === // ==================== - // Process each row as needed - rows = rows.map(row => row.split(',')[0].trim()); + // Process each row as needed, in this case, get first column value + rows = rows.map(row => row[0]); console.log(`Number of Rows Found: ${rows.length}`); @@ -83,7 +83,7 @@ function readCsv(fileName, hasHeader = false) { const rows = fs.readFileSync(fileName, { encoding: 'utf8' }) .split('\n') - .map(s => s.trim()) || []; + .map(r => r.split(",").map(s => s.trim())) || []; - return hasHeader && rows.length > 0 ? rows.splice(1) : rows; + return hasHeader ? rows.splice(1) : rows; } \ No newline at end of file diff --git a/examples/getAllTasks.js b/examples/getAllTasks.js index aeb8d0c..f06ade8 100644 --- a/examples/getAllTasks.js +++ b/examples/getAllTasks.js @@ -1,13 +1,14 @@ -const scaleapi = require('../lib/scaleapi.js'); // Change to just "scaleapi" for your project +const scaleapi = require('scaleapi'); // Change to "../lib/scaleapi.js" if you intend to run in this repo const fs = require('fs'); +const maxTasksReturnedPerCall = 100; + // HOW IT WORKS: // Given a list of search filters ("PARAMS"), it will page through tasks and write // the output to a JSON file. - // INPUT PARAMETERS: -const API_KEY = 'live_xxxxx'; +const API_KEY = 'live_xxx'; const MAX_TASKS_TO_RETURN = 100000; // Get up to the n most recently created tasks @@ -29,26 +30,19 @@ const PARAMS = { // All params optional // Get list of task objects let getTasks = async function(client, maxTasksToReturn = 1000000, params = {}) { // Initialize everything - const maxTasksReturnedPerCall = 100; let lastPageCount = maxTasksReturnedPerCall; let output = []; // Go through page by page - while (lastPageCount == maxTasksReturnedPerCall && output.length < maxTasksToReturn) { - - // Skip ahead (offset) by the number of tasks already pulled - params.offset = output.length; - - // Support asking for small number of tasks (like 5, 10, 50) - params.limit = maxTasksReturnedPerCall; - if (maxTasksToReturn - output.length < maxTasksReturnedPerCall) { - params.limit = maxTasksToReturn - output.length - } - + while (lastPageCount === maxTasksReturnedPerCall && output.length < maxTasksToReturn) { try { // fetch some tasks let tasks = await new Promise((resolve, reject) => { - client.tasks(params, (err, tasklist) => { + client.tasks({ + ...params, + offset: output.length, + limit: maxTasksToReturn - output.length < maxTasksReturnedPerCall ? maxTasksToReturn - output.length : maxTasksReturnedPerCall + }, (err, tasklist) => { if (err) { console.error(err); reject(err);