Node.js pagination error

Albusis · December 30, 2024, 12:32pm

Hi,

I have a problem. I have an API program that accesses the HubSpot file system and returns some replies. It’s a big reply, so I try to use pagination. Sadly when i request a pagination token it comes as a string, in this case, “AAAACg”. When it then follows to page 2 it gives an error. It takes the first 10 objects it gets and successfully inserts it into Google Sheets, and when it wants to get page 2, it stops. The program is in node.js and i run on my VS Code Studio. What could be the problem? I’ve spent days and even hired 14 different upwork freelancers. Nobody was able to move it.

I wanted to contact Slack HS Developers, but the Slack seems to be turned off.

The error this.

Google Sheets authenticated!
HubSpot API Request Parameters:
{ after: undefined }
Next 'after' token: AAAACg
Type of 'after' token before conversions: string
Decoded 'after' token: NaN
Data saved to Google Sheets!
Error fetching items: HTTP-Code: 400
Message: An error occurred.
Body: {"status":"error","message":"Invalid input JSON on line -1, column -1: Cannot deserialize value of type `long` from String \"AAAACg\": not a valid `long` value","correlationId":"e2f4e619-0883-4174-89ac-5f347e01321a"}
Headers: {"access-control-allow-credentials":"false","cf-cache-status":"DYNAMIC","cf-ray":"8fa200e40a88b391-PRG","connection":"keep-alive","content-length":"216","content-type":"application/json;charset=utf-8","date":"Mon, 30 Dec 2024 12:23:51 GMT","nel":"{\"success_fraction\":0.01,\"report_to\":\"cf-nel\",\"max_age\":604800}","report-to":"{\"endpoints\":[{\"url\":\"https:\\/\\/a.nel.cloudflare.com\\/report\\/v4?s=rH9fhc3xl5oTsShl9DBuI26AZMxM48fWHFvPFBaU1Yfbr8dA0TFqiyeeq6ABnQMFsUwlw9kHgd8xatbDCst4VrPq0WsHKvqn08aUzMqDdtYC0P1pfd9Nle2CCdzX6ENs\"}],\"group\":\"cf-nel\",\"max_age\":604800}","server":"cloudflare","set-cookie":"__cf_bm=o3oVSzuDJGqTZi.n5NNJgRzHxNReqI27loqAA1dRt5E-1735561431-1.0.1.1-n2cCtOuB0r_IoO_ZLQ_0VICD8tl92_MLhLrEssigvWE5rNYOSKqfjlOImn.KRGRYfoTUUGkRbcNKptG4dAQl7A; path=/; expires=Mon, 30-Dec-24 12:53:51 GMT; domain=.hubapi.com; HttpOnly; Secure; SameSite=None","strict-transport-security":"max-age=31536000; includeSubDomains; preload","vary":"origin, Accept-Encoding","x-content-type-options":"nosniff","x-hubspot-correlation-id":"e2f4e619-0883-4174-89ac-5f347e01321a","x-hubspot-ratelimit-daily":"625000","x-hubspot-ratelimit-daily-remaining":"622290","x-hubspot-ratelimit-interval-milliseconds":"10000","x-hubspot-ratelimit-max":"190","x-hubspot-ratelimit-remaining":"188","x-hubspot-ratelimit-secondly":"19","x-hubspot-ratelimit-secondly-remaining":"18"}
API Response Body: No response body
Fetched 10 items in total.

My code is this.

require('dotenv').config(); // Load .env file
const { google } = require("googleapis");
const hubspot = require('@hubspot/api-client');

// Initialize the HubSpot client with the token from .env
const hubspotClient = new hubspot.Client({
 accessToken: process.env.HUBSPOT_API_TOKEN,
});

// Authenticate Google Sheets API
async function authenticateGoogleSheets() {
 const auth = new google.auth.GoogleAuth({
 keyFile: "credentials.json", // Path to your service account key file
 scopes: ["https://www.googleapis.com/auth/spreadsheets"],
 });
 return google.sheets({ version: "v4", auth });
}

async function appendToGoogleSheet(sheets, spreadsheetId, range, values) {
 try {
 const result = await sheets.spreadsheets.values.append({
 spreadsheetId,
 range,
 valueInputOption: "USER_ENTERED",
 resource: { values },
 });
 console.log("Data saved to Google Sheets!");
 return result;
 } catch (error) {
 console.error("Error appending to Google Sheet:", error.message);
 throw error;
 }
}

// Function to convert bytes to kB or MB
function formatSize(bytes) {
 if (bytes < 1024) {
 return `${bytes} B`; // Bytes
 } else if (bytes < 1024 * 1024) {
 return `${(bytes / 1024).toFixed(2)} kB`; // Kilobytes
 } else {
 return `${(bytes / (1024 * 1024)).toFixed(2)} MB`; // Megabytes
 }
}

async function fetchAllItems(parentFolderId = undefined, sheets, spreadsheetId) {
 const results = [];
 let after = undefined; // Pagination cursor

 do {
 try {
 // Call HubSpot API to search files and folders
 const apiResponse = await hubspotClient.files.filesApi.doSearch(
 undefined, // name
 undefined, // path
 undefined, // url
 undefined, // type
 undefined, // size
 after // Pagination cursor
 );

 // Log the request details
 console.log("HubSpot API Request Parameters:");
 console.log({
 after: after, // Pagination token
 });

 const items = apiResponse.results;
 after = apiResponse.paging?.next?.after; // Keep the token as a string or undefined

 console.log(apiResponse);

 // Debugging: Log the `after` value and its type
 console.log("Next 'after' token:", after);
 console.log("Type of 'after' token before conversions:", typeof after);

 if (!after) {
 console.log('No more items to fetch.');
 break;
 }

 // Process each item and extract only the desired properties
 const filteredItems = items.map((item) => [
 item.name,
 item.path,
 item.url,
 item.type,
 item.size ? formatSize(item.size) : "N/A" // Convert size to kB or MB
 ]);

 // Append filtered items to Google Sheets in Tab '20'
 await appendToGoogleSheet(sheets, spreadsheetId, "20!A2", filteredItems);

 for (const item of items) {
 results.push(item);

 // If the item is a folder, fetch its contents recursively
 if (item.type === 'FOLDER') {
 const subItems = await fetchAllItems(item.id, sheets, spreadsheetId);
 results.push(...subItems);
 }
 }
 } catch (error) {
 console.error("Error fetching items:", error.message);
 console.error("API Response Body:", error.response?.data || "No response body");
 break; // Stop further processing on error
 }
 } while (after); // Continue fetching until all pages are retrieved

 return results;
}

// Main function to fetch all items starting from the root
(async () => {
 if (!process.env.HUBSPOT_API_TOKEN) {
 console.error("HUBSPOT_API_TOKEN is missing. Check your .env file.");
 process.exit(1); // Exit if token is not set
 }

 const spreadsheetId = "1J6lzh42NB7quPuaUUkdj-W8yFAsuaaGnSQNXQ8P_DNk"; // Google Sheets ID

 try {
 const sheets = await authenticateGoogleSheets(); // Authenticate Sheets API
 console.log("Google Sheets authenticated!");

 const allItems = await fetchAllItems(undefined, sheets, spreadsheetId); // Start from the root folder
 console.log(`Fetched ${allItems.length} items in total.`);
 } catch (error) {
 console.error("Error fetching items:", error.message);
 }
})();

I think the issue is that it returns a string, which by GitHub documentation i found is what we should expect, but when sending it back for page 2 it says it wants long. And I wasn’t able to change it from string to long. What do I do?

evaldas · December 30, 2024, 3:20pm

Hi @Albusis,

Not entirely sure this will work, but have you tried decoding the “AAAACg” string (which is likely base64-encoded) into long integer?

function decodeAfterToken(after) {
 if (!after) return undefined;
 const buf = Buffer.from(after, 'base64');
 return buf.readInt32BE(0);
}

Then in your apiResponse const:

const apiResponse = await hubspotClient.files.filesApi.doSearch(
 undefined,
 undefined, 
 undefined,
 undefined,
 undefined,
 decodeAfterToken(after)
);

SteveHTM · December 30, 2024, 7:10pm

@Albusis - In my experience there is never any conversion reqired for the ‘after’ values.. It is an usually an opaque property that you simply pass to the next call to get successive results.

The documentation for the base call is here: Accounts Dashboard | HubSpot

And it’s pretty clearly a string value there.

Best of luck!

Steve

Albusis · December 30, 2024, 7:35pm

Hello @evaldas , I didn’t have the opportunity to test it, but one of the 14 freelancers I hired actually discovered that I’m asking only maybe 5 parameters, where there are 10 required. So when we started requesting all required parameters it started working normally.

For everyone with the same problem, this is my final code:

require('dotenv').config(); // Load .env file
const { google } = require("googleapis");
const hubspot = require('@hubspot/api-client');

// Initialize the HubSpot client with the token from .env
const hubspotClient = new hubspot.Client({
 accessToken: process.env.HUBSPOT_API_TOKEN,
});

// Authenticate Google Sheets API
async function authenticateGoogleSheets() {
 const auth = new google.auth.GoogleAuth({
 keyFile: "credentials.json", // Path to your service account key file
 scopes: ["https://www.googleapis.com/auth/spreadsheets"],
 });
 return google.sheets({ version: "v4", auth });
}

async function appendToGoogleSheet(sheets, spreadsheetId, range, values) {
 try {
 const result = await sheets.spreadsheets.values.append({
 spreadsheetId,
 range,
 valueInputOption: "USER_ENTERED",
 resource: { values },
 });
 console.log("Data saved to Google Sheets!");
 return result;
 } catch (error) {
 console.error("Error appending to Google Sheet:", error.message);
 throw error;
 }
}

// Function to convert bytes to kB or MB
function formatSize(bytes) {
 if (bytes < 1024) {
 return `${bytes} B`; // Bytes
 } else if (bytes < 1024 * 1024) {
 return `${(bytes / 1024).toFixed(2)} kB`; // Kilobytes
 } else {
 return `${(bytes / (1024 * 1024)).toFixed(2)} MB`; // Megabytes
 }
}

async function fetchAllItems(parentFolderId = undefined, sheets, spreadsheetId) {
 const results = [];
 let after = undefined; // Pagination cursor
 const properties = undefined;
 const before = undefined;
 const limit = undefined;
 const sort = undefined;
 const id = undefined;
 const createdAt = undefined;
 const createdAtLte = undefined;
 const createdAtGte = undefined;
 const updatedAt = undefined;
 const updatedAtLte = undefined;
 const updatedAtGte = undefined;
 const name = undefined;
 const path = undefined;
 const size = undefined;
 const height = undefined;
 const width = undefined;
 const encoding = undefined;
 const type = undefined;
 const extension = undefined;
 const url = undefined;
 const isUsableInContent = undefined;
 const allowsAnonymousAccess = undefined;

 do {
 try {
 // Call HubSpot API to search files and folders
 const apiResponse = await hubspotClient.files.filesApi.doSearch(properties, after, before, limit, sort, id, createdAt, createdAtLte, createdAtGte, updatedAt, updatedAtLte, updatedAtGte, name, path, parentFolderId, size, height, width, encoding, type, extension, url, isUsableInContent, allowsAnonymousAccess);

 // Log the request details
 console.log("HubSpot API Request Parameters:");
 console.log({
 after: after, // Pagination token
 });

 const items = apiResponse.results;
 after = apiResponse.paging?.next?.after; // Keep the token as a string or undefined

 // Debugging: Log the `after` value and its type
 console.log("Next 'after' token:", after);
 console.log("Type of 'after' token before conversions:", typeof after);

 // Process each item and extract only the desired properties
 const filteredItems = items.map((item) => [
 item.name,
 item.path,
 item.url,
 item.type,
 item.size ? formatSize(item.size) : "N/A" // Convert size to kB or MB
 ]);

 // Append filtered items to Google Sheets in Tab '20'
 await appendToGoogleSheet(sheets, spreadsheetId, "20!A2", filteredItems);

 for (const item of items) {
 results.push(item);

 // If the item is a folder, fetch its contents recursively
 if (item.type === 'FOLDER') {
 const subItems = await fetchAllItems(item.id, sheets, spreadsheetId);
 results.push(...subItems);
 }
 }
 if (!after) {
 console.log('No more items to fetch.');
 break;
 }
 } catch (error) {
 console.error("Error fetching items:", error.message);
 console.error("API Response Body:", error.response?.data || "No response body");
 break; // Stop further processing on error
 }
 } while (after); // Continue fetching until all pages are retrieved

 return results;
}

// Main function to fetch all items starting from the root
(async () => {
 if (!process.env.HUBSPOT_API_TOKEN) {
 console.error("HUBSPOT_API_TOKEN is missing. Check your .env file.");
 process.exit(1); // Exit if token is not set
 }

 const spreadsheetId = "1J6lzh42NB7quPuaUUkdj-W8yFAsuaaGnSQNXQ8P_DNk"; // Google Sheets ID

 try {
 const sheets = await authenticateGoogleSheets(); // Authenticate Sheets API
 console.log("Google Sheets authenticated!");

 const allItems = await fetchAllItems(undefined, sheets, spreadsheetId); // Start from the root folder
 console.log(`Fetched ${allItems.length} items in total.`);
 } catch (error) {
 console.error("Error fetching items:", error.message);
 }
})();

evaldas · December 30, 2024, 7:43pm

Ah, that will do it. Thanks for sharing the working version

Topic		Replies	Views
Hubspot and Power BI Coding Issue APIs & Integrations starter	2	647	August 22, 2025
API Requests Issues - Private App (Error: 403) APIs & Integrations python	1	1491	October 22, 2024
Unable to fetch more than 100 records from Hubspot API APIs & Integrations professional	10	7548	June 1, 2021
pagination issue APIs & Integrations	2	949	October 8, 2022
How to paginate with cursor? APIs & Integrations professional , contacts , timeline-and-extensions-apis , developer-tools	10	10063	August 10, 2023

Node.js pagination error

Related topics