playwright#firefox TypeScript Examples

The following examples show how to use playwright#firefox. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: get-client-rectangle-of.firefox.test.ts    From playwright-fluent with MIT License 6 votes vote down vote up
describe('get client rectangle', (): void => {
  let browser: Browser | undefined = undefined;
  // eslint-disable-next-line @typescript-eslint/no-empty-function
  beforeEach((): void => {});
  afterEach(async (): Promise<void> => {
    if (browser) {
      await browser.close();
    }
  });
  test('should return an error when selector is not found - firefox', async (): Promise<void> => {
    // Given
    browser = await firefox.launch({ headless: true });
    const browserContext = await browser.newContext({ viewport: null });
    const page = await browserContext.newPage();

    // When

    // Then
    const expectedError = new Error(
      'page.$eval: Error: failed to find element matching selector "foobar"',
    );
    await SUT.getClientRectangleOf('foobar', page).catch((error): void =>
      expect(error).toMatchObject(expectedError),
    );
  });
});
Example #2
Source File: inject-cursor.firefox.test.ts    From playwright-fluent with MIT License 6 votes vote down vote up
describe('inject-cursor', (): void => {
  let browser: Browser | undefined = undefined;
  // eslint-disable-next-line @typescript-eslint/no-empty-function
  beforeEach((): void => {});
  afterEach(async (): Promise<void> => {
    if (browser) {
      await browser.close();
    }
  });
  test('should show cursor on the page on firefox', async (): Promise<void> => {
    // Given
    const url = 'https://reactstrap.github.io/components/form';
    browser = await firefox.launch({ headless: true });
    const context = await browser.newContext({ viewport: null });
    const page = await context.newPage();
    await page.goto(url);

    // When
    await SUT.injectCursor(page);
    await SUT.injectCursor(page);
    await SUT.injectCursor(page);
    await SUT.injectCursor(page);

    // Then
    const cursorExists = await exists('playwright-mouse-pointer', page);
    expect(cursorExists).toBe(true);
  });
});
Example #3
Source File: get-client-rectangle-of-handle.firefox.test.ts    From playwright-fluent with MIT License 6 votes vote down vote up
// TODO: re-enable these tests on v1.0.0
describe.skip('get client rectangle of an element handle', (): void => {
  let browser: Browser | undefined = undefined;
  // eslint-disable-next-line @typescript-eslint/no-empty-function
  beforeEach((): void => {});
  afterEach(async (): Promise<void> => {
    if (browser) {
      await browser.close();
    }
  });

  test('should return Client Rectangle - firefox', async (): Promise<void> => {
    // Given
    browser = await firefox.launch({ headless: true });
    const browserContext = await browser.newContext({ viewport: null });
    const page = await browserContext.newPage();
    const url = `file:${path.join(__dirname, 'get-client-rectangle-of-handle.test.html')}`;
    await page.goto(url);
    await sleep(1000);

    // When
    const handle = await page.$('#foo');
    const result = await SUT.getClientRectangleOfHandle(handle);

    // Then
    const expectedClientRectangle: SerializableDOMRect = {
      bottom: 32,
      height: 21,
      left: 12,
      right: 32,
      top: 11,
      width: 20,
      x: 12, // left
      y: 11, // top
    };
    expect(result).not.toBe(null);
    expect(result).toMatchObject(expectedClientRectangle);
  });
});
Example #4
Source File: get-focused-handle.firefox.test.ts    From playwright-fluent with MIT License 6 votes vote down vote up
describe('get-focused-handle', (): void => {
  let browser: Browser | undefined = undefined;
  // eslint-disable-next-line @typescript-eslint/no-empty-function
  beforeEach((): void => {});
  afterEach(async (): Promise<void> => {
    if (browser) {
      await browser.close();
    }
  });

  test('should return handle when selector exists on the page - firefox', async (): Promise<void> => {
    // Given
    browser = await firefox.launch({ headless: true });
    const context = await browser.newContext({ viewport: null });
    const page = await context.newPage();

    // When
    const result = await SUT.getFocusedHandle(page);

    // Then
    expect(result).toBeDefined();
  });
});
Example #5
Source File: get-viewport-rectangle-of.firefox.test.ts    From playwright-fluent with MIT License 6 votes vote down vote up
describe('get viewport rectangle of page', (): void => {
  let browser: Browser | undefined = undefined;
  // eslint-disable-next-line @typescript-eslint/no-empty-function
  beforeEach((): void => {});
  afterEach(async (): Promise<void> => {
    if (browser) {
      await browser.close();
    }
  });
  test.skip('should return defaultViewport - firefox', async (): Promise<void> => {
    // Given
    browser = await firefox.launch({ headless: false });
    const browserContext = await browser.newContext();
    const page = await browserContext.newPage();

    // When
    const result = await SUT.getViewportRectangleOf(page);

    // Then
    const defaultViewportRectangle: ViewportRect = {
      height: 600,
      offsetLeft: 0,
      offsetTop: 0,
      pageLeft: 0,
      pageTop: 0,
      scale: 1,
      width: 800,
    };
    expect(result).toBeDefined();
    expect(result).toMatchObject(defaultViewportRectangle);
  });
});
Example #6
Source File: server.ts    From Assistive-Webdriver with MIT License 5 votes vote down vote up
browsers: { [key: string]: BrowserType<Browser> } = {
  chromium,
  webkit,
  firefox
}
Example #7
Source File: launch-browser.ts    From playwright-fluent with MIT License 5 votes vote down vote up
export async function launchBrowser(name: BrowserName, options: LaunchOptions): Promise<Browser> {
  switch (name) {
    case 'chrome': {
      if (options && options.executablePath !== undefined) {
        const browser = await chromium.launch(options);
        return browser;
      }
      {
        const chromeOptions: LaunchOptions = {
          ...options,
          executablePath: getChromePath(),
        };
        const browser = await chromium.launch(chromeOptions);
        return browser;
      }
    }

    case 'chrome-canary': {
      if (options && options.executablePath !== undefined) {
        const browser = await chromium.launch(options);
        return browser;
      }
      {
        const chromeOptions: LaunchOptions = {
          ...options,
          executablePath: getChromeCanaryPath(),
        };
        const browser = await chromium.launch(chromeOptions);
        return browser;
      }
    }

    case 'msedge': {
      if (options && options.executablePath !== undefined) {
        const browser = await chromium.launch(options);
        return browser;
      }
      {
        const chromeOptions: LaunchOptions = {
          ...options,
          executablePath: getEdgePath(),
        };
        const browser = await chromium.launch(chromeOptions);
        return browser;
      }
    }

    case 'chromium': {
      const browser = await chromium.launch(options);
      return browser;
    }

    case 'firefox': {
      const browser = await firefox.launch(options);
      return browser;
    }

    case 'webkit': {
      const browser = await webkit.launch(options);
      return browser;
    }

    default:
      throw new Error(
        `Browser named '${name}' is unknown. It should be one of 'chrome', 'chromium', 'chrome-canary', 'firefox', 'webkit'`,
      );
  }
}
Example #8
Source File: show-mouse-position.firefox.test.ts    From playwright-fluent with MIT License 5 votes vote down vote up
describe('show-mouse-position', (): void => {
  let browser: Browser | undefined = undefined;
  // eslint-disable-next-line @typescript-eslint/no-empty-function
  beforeEach((): void => {});
  afterEach(async (): Promise<void> => {
    if (browser) {
      await browser.close();
    }
  });
  test('should show cursor on the page on firefox', async (): Promise<void> => {
    // Given
    const url = 'https://reactstrap.github.io/components/form';
    browser = await firefox.launch({ headless: true });
    const context = await browser.newContext({ viewport: null });
    const page = await context.newPage();

    // When
    await SUT.showMousePosition(page);
    await page.goto(url);

    // Then
    const cursorExists = await exists('playwright-mouse-pointer', page);
    expect(cursorExists).toBe(true);
  });

  test('should show cursor on navigating to another page on firefox', async (): Promise<void> => {
    // Given
    const url = 'https://reactstrap.github.io/components/form';
    browser = await firefox.launch({ headless: true });
    const context = await browser.newContext({ viewport: null });
    const page = await context.newPage();

    // When
    await SUT.showMousePosition(page);
    await page.goto(url);
    await page.goto('https://google.com');

    // Then
    const cursorExists = await exists('playwright-mouse-pointer', page);
    expect(cursorExists).toBe(true);
  });
});
Example #9
Source File: is-handle-moving.firefox.test.ts    From playwright-fluent with MIT License 5 votes vote down vote up
// TODO: re-enable these tests on v1.0.0
describe.skip('handle is moving', (): void => {
  let browser: Browser | undefined = undefined;
  // eslint-disable-next-line @typescript-eslint/no-empty-function
  beforeEach((): void => {});
  afterEach(async (): Promise<void> => {
    if (browser) {
      await browser.close();
    }
  });
  test('should detect that selector is moving - firefox', async (): Promise<void> => {
    // Given
    browser = await firefox.launch({ headless: true });
    const browserContext = await browser.newContext({ viewport: null });
    const page = await browserContext.newPage();
    await showMousePosition(page);
    const url = `file:${path.join(__dirname, 'is-handle-moving.test1.html')}`;
    await page.goto(url);
    await sleep(100); // wait for the animation to be started

    // When
    const selector = '#moving';
    const handle = await page.$(selector);
    const isMoving = await SUT.isHandleMoving(handle);

    // Then
    expect(isMoving).toBe(true);
  });

  test('should detect that selector is not moving - firefox', async (): Promise<void> => {
    // Given
    browser = await firefox.launch({ headless: true });
    const browserContext = await browser.newContext({ viewport: null });
    const page = await browserContext.newPage();
    await showMousePosition(page);
    const url = `file:${path.join(__dirname, 'is-handle-moving.test2.html')}`;
    await page.goto(url);
    await sleep(2000); // wait twice the animation duration

    // When
    const selector = '#moving';
    const handle = await page.$(selector);
    const isMoving = await SUT.isHandleMoving(handle);

    // Then
    expect(isMoving).toBe(false);
  });
});
Example #10
Source File: get-handle-of.firefox.test.ts    From playwright-fluent with MIT License 5 votes vote down vote up
describe('get-handle-of', (): void => {
  let browser: Browser | undefined = undefined;
  // eslint-disable-next-line @typescript-eslint/no-empty-function
  beforeEach((): void => {});
  afterEach(async (): Promise<void> => {
    if (browser) {
      await browser.close();
    }
  });

  test('should return handle when selector exists on the page - firefox', async (): Promise<void> => {
    // Given
    browser = await firefox.launch({ headless: true });
    const context = await browser.newContext({ viewport: null });
    const page = await context.newPage();

    // When
    const result = await SUT.getHandleOf('body', page, defaultWaitUntilOptions);

    // Then
    expect(result).toBeDefined();
  });

  test('should throw an error when selector does not exist on the page - firefox', async (): Promise<void> => {
    // Given
    browser = await firefox.launch({ headless: true });
    const context = await browser.newContext({ viewport: null });
    const page = await context.newPage();
    const options: WaitUntilOptions = {
      ...defaultWaitUntilOptions,
      timeoutInMilliseconds: 2000,
    };

    // When
    let result: Error | undefined = undefined;
    try {
      await SUT.getHandleOf('foobar', page, options);
    } catch (error) {
      result = error as Error;
    }

    // Then
    expect(result && result.message).toContain("Selector 'foobar' was not found in DOM");
  });
});
Example #11
Source File: hover-on-selector.firefox.test.ts    From playwright-fluent with MIT License 5 votes vote down vote up
// TODO: re-enable these tests on v1.0.0
describe.skip('hover on selector', (): void => {
  let browser: Browser | undefined = undefined;

  // eslint-disable-next-line @typescript-eslint/no-empty-function
  beforeEach((): void => {});

  afterEach(async (): Promise<void> => {
    if (browser) {
      await browser.close();
    }
  });

  test('should wait for the selector to exists before hovering - firefox', async (): Promise<void> => {
    // Given
    browser = await firefox.launch({ headless: false });
    const browserContext = await browser.newContext({ viewport: null });
    const page = await browserContext.newPage();
    await showMousePosition(page);
    const url = `file:${path.join(__dirname, 'hover-on-selector.test.html')}`;
    await page.goto(url);

    const selector = '#dynamically-added';
    let handle = await page.$(selector);
    const isSelectorVisibleBeforeScroll = await isHandleVisible(handle, defaultVerboseOptions);

    const options: HoverOptions = {
      ...defaultHoverOptions,
    };

    // When
    await SUT.hoverOnSelector(selector, page, options);
    handle = await page.$(selector);
    const isSelectorVisibleAfterScroll = await isHandleVisible(handle, defaultVerboseOptions);

    // Then
    expect(isSelectorVisibleBeforeScroll).toBe(false);
    expect(isSelectorVisibleAfterScroll).toBe(true);

    const mousePositionClientRectangle = await getClientRectangleOf(
      'playwright-mouse-pointer',
      page,
    );
    const mouseX = mousePositionClientRectangle.left + mousePositionClientRectangle.width / 2;
    const mouseY = mousePositionClientRectangle.top + mousePositionClientRectangle.height / 2;

    const currentClientRectangle = await getClientRectangleOf(selector, page);
    const expectedX = currentClientRectangle.left + currentClientRectangle.width / 2;
    const expectedY = currentClientRectangle.top + currentClientRectangle.height / 2;
    expect(Math.abs(mouseX - expectedX)).toBeLessThanOrEqual(1);
    expect(Math.abs(mouseY - expectedY)).toBeLessThanOrEqual(1);
  });
});
Example #12
Source File: ctor-1.test.ts    From playwright-fluent with MIT License 5 votes vote down vote up
describe('Playwright Fluent - ctor usage', (): void => {
  // eslint-disable-next-line @typescript-eslint/no-empty-function
  beforeEach((): void => {});
  test('should take existing browser and page instance of chromium', async (): Promise<void> => {
    // Given
    const browser = await chromium.launch({ headless: true });
    const context = await browser.newContext();
    const url = `file:${path.join(__dirname, 'ctor.test.html')}`;
    const page = await context.newPage();
    await page.goto(url);

    // When
    const p = new PlaywrightFluent(browser, page);

    // Then
    expect(p.currentBrowser()).toBe(browser);
    expect(p.currentPage()).toBe(page);
    expect(p.currentFrame()).toBeUndefined();
    await browser.close();
  });

  test('should take existing browser and page instance of firefox', async (): Promise<void> => {
    // Given
    const browser = await firefox.launch({ headless: true });
    const context = await browser.newContext();
    // const url = `file:${path.join(__dirname, 'ctor.test.html')}`;
    const page = await context.newPage();
    await page.goto('https://google.com');

    // When
    const p = new PlaywrightFluent(browser, page);

    // Then
    expect(p.currentBrowser()).toBe(browser);
    expect(p.currentPage()).toBe(page);
    expect(p.currentFrame()).toBeUndefined();
    await browser.close();
  });

  test('should take existing browser and page instance of webkit', async (): Promise<void> => {
    // Given
    const browser = await webkit.launch({ headless: true });
    const context = await browser.newContext();
    const url = `file:${path.join(__dirname, 'ctor.test.html')}`;
    const page = await context.newPage();
    await page.goto(url);

    // When
    const p = new PlaywrightFluent(browser, page);

    // Then
    expect(p.currentBrowser()).toBe(browser);
    expect(p.currentPage()).toBe(page);
    expect(p.currentFrame()).toBeUndefined();
    await browser.close();
  });

  test.skip('should take existing browser and page instance of firefox', async (): Promise<void> => {
    // Given
    const browser = await firefox.launch({ headless: true });
    const context = await browser.newContext();
    const url = `file:${path.join(__dirname, 'ctor.test.html')}`;
    const page = await context.newPage();
    await page.goto(url);

    // When
    const p = new PlaywrightFluent(browser, page);

    // Then
    expect(p.currentBrowser()).toBe(browser);
    expect(p.currentPage()).toBe(page);
    expect(p.currentFrame()).toBeUndefined();
    await browser.close();
  });
});
Example #13
Source File: is-handle-visible.firefox.test.ts    From playwright-fluent with MIT License 4 votes vote down vote up
describe('handle is visible', (): void => {
  let browser: Browser | undefined = undefined;
  // eslint-disable-next-line @typescript-eslint/no-empty-function
  beforeEach((): void => {});
  afterEach(async (): Promise<void> => {
    if (browser) {
      await browser.close();
    }
  });

  test('should return false when selector is hidden - firefox', async (): Promise<void> => {
    // Given
    browser = await firefox.launch({ headless: true });
    const browserContext = await browser.newContext({ viewport: null });
    const page = await browserContext.newPage();
    const url = `file:${path.join(__dirname, 'is-handle-visible.test.html')}`;
    await page.goto(url);
    await sleep(1000);

    const handle = await page.$('#hidden');

    // When
    const result = await SUT.isHandleVisible(handle, defaultVerboseOptions);

    // Then
    expect(handle).toBeDefined();
    expect(result).toBe(false);
  });

  test('should return true when selector is visible', async (): Promise<void> => {
    // Given
    browser = await firefox.launch({ headless: true });
    const browserContext = await browser.newContext({ viewport: null });
    const page = await browserContext.newPage();
    const url = `file:${path.join(__dirname, 'is-handle-visible.test.html')}`;
    await page.goto(url);
    await sleep(1000);

    const handle = await page.$('#visible');

    // When
    const result = await SUT.isHandleVisible(handle, defaultVerboseOptions);

    // Then
    expect(handle).toBeDefined();
    expect(result).toBe(true);
  });

  test('should return false when selector is transparent', async (): Promise<void> => {
    // Given
    browser = await firefox.launch({ headless: true });
    const browserContext = await browser.newContext({ viewport: null });
    const page = await browserContext.newPage();
    const url = `file:${path.join(__dirname, 'is-handle-visible.test.html')}`;
    await page.goto(url);
    await sleep(1000);

    const handle = await page.$('#transparent');

    // When
    const result = await SUT.isHandleVisible(handle, defaultVerboseOptions);

    // Then
    expect(handle).toBeDefined();
    expect(result).toBe(false);
  });

  test('should return false when selector is out of screen', async (): Promise<void> => {
    // Given
    browser = await firefox.launch({ headless: true });
    const browserContext = await browser.newContext({ viewport: null });
    const page = await browserContext.newPage();
    const url = `file:${path.join(__dirname, 'is-handle-visible.test.html')}`;
    await page.goto(url);
    await sleep(1000);

    const handle = await page.$('#out-of-screen');

    // When
    const result = await SUT.isHandleVisible(handle, defaultVerboseOptions);

    // Then
    expect(handle).toBeDefined();
    expect(result).toBe(false);
  });

  test('should return true when selector is visible but out of viewport', async (): Promise<void> => {
    // Given
    browser = await firefox.launch({ headless: true });
    const browserContext = await browser.newContext({ viewport: null });
    const page = await browserContext.newPage();
    const url = `file:${path.join(__dirname, 'is-handle-visible.test.html')}`;
    await page.goto(url);
    await sleep(1000);

    const handle = await page.$('#out-of-viewport');

    // When
    const result = await SUT.isHandleVisible(handle, defaultVerboseOptions);

    // Then
    expect(handle).toBeDefined();
    expect(result).toBe(true);
  });
});
Example #14
Source File: is-handle-visible-in-viewport.firefox.test.ts    From playwright-fluent with MIT License 4 votes vote down vote up
describe('handle is visible in viewport', (): void => {
  let browser: Browser | undefined = undefined;
  // eslint-disable-next-line @typescript-eslint/no-empty-function
  beforeEach((): void => {});
  afterEach(async (): Promise<void> => {
    if (browser) {
      await browser.close();
    }
  });

  test('should return false when selector is hidden - firefox', async (): Promise<void> => {
    // Given
    browser = await firefox.launch({ headless: true });
    const browserContext = await browser.newContext({ viewport: null });
    const page = await browserContext.newPage();
    const url = `file:${path.join(__dirname, 'is-handle-visible-in-viewport.test.html')}`;
    await page.goto(url);
    await sleep(1000);

    const handle = await page.$('#hidden');

    // When
    const result = await SUT.isHandleVisibleInViewport(handle, defaultVerboseOptions);

    // Then
    expect(handle).toBeDefined();
    expect(result).toBe(false);
  });

  test('should return true when selector is visible in viewport', async (): Promise<void> => {
    // Given
    browser = await firefox.launch({ headless: true });
    const browserContext = await browser.newContext({ viewport: null });
    const page = await browserContext.newPage();
    const url = `file:${path.join(__dirname, 'is-handle-visible-in-viewport.test.html')}`;
    await page.goto(url);
    await sleep(1000);

    const handle = await page.$('#visible');

    // When
    const result = await SUT.isHandleVisibleInViewport(handle, defaultVerboseOptions);

    // Then
    expect(handle).toBeDefined();
    expect(result).toBe(true);
  });

  test('should return false when selector is transparent', async (): Promise<void> => {
    // Given
    browser = await firefox.launch({ headless: true });
    const browserContext = await browser.newContext({ viewport: null });
    const page = await browserContext.newPage();
    const url = `file:${path.join(__dirname, 'is-handle-visible-in-viewport.test.html')}`;
    await page.goto(url);
    await sleep(1000);

    const handle = await page.$('#transparent');

    // When
    const result = await SUT.isHandleVisibleInViewport(handle, defaultVerboseOptions);

    // Then
    expect(handle).toBeDefined();
    expect(result).toBe(false);
  });

  test('should return false when selector is out of screen', async (): Promise<void> => {
    // Given
    browser = await firefox.launch({ headless: true });
    const browserContext = await browser.newContext({ viewport: null });
    const page = await browserContext.newPage();
    const url = `file:${path.join(__dirname, 'is-handle-visible-in-viewport.test.html')}`;
    await page.goto(url);
    await sleep(1000);

    const handle = await page.$('#out-of-screen');

    // When
    const result = await SUT.isHandleVisibleInViewport(handle, defaultVerboseOptions);

    // Then
    expect(handle).toBeDefined();
    expect(result).toBe(false);
  });

  test('should return false when selector is out of viewport', async (): Promise<void> => {
    // Given
    browser = await firefox.launch({ headless: true });
    const browserContext = await browser.newContext({ viewport: null });
    const page = await browserContext.newPage();
    const url = `file:${path.join(__dirname, 'is-handle-visible-in-viewport.test.html')}`;
    await page.goto(url);
    await sleep(1000);

    const handle = await page.$('#out-of-viewport');

    // When
    const result = await SUT.isHandleVisibleInViewport(handle, defaultVerboseOptions);

    // Then
    expect(handle).toBeDefined();
    expect(result).toBe(false);
  });
});
Example #15
Source File: scroll-to-handle.firefox.test.ts    From playwright-fluent with MIT License 4 votes vote down vote up
// TODO: re-enable these tests on v1.0.0
describe.skip('scroll to handle', (): void => {
  let browser: Browser | undefined = undefined;

  // eslint-disable-next-line @typescript-eslint/no-empty-function
  beforeEach((): void => {});

  afterEach(async (): Promise<void> => {
    if (browser) {
      await browser.close();
    }
  });
  test('should scroll to a selector that is out of viewport - firefox', async (): Promise<void> => {
    // Given
    browser = await firefox.launch({ headless: true });
    const browserContext = await browser.newContext({ viewport: null });
    const page = await browserContext.newPage();
    await showMousePosition(page);
    const url = `file:${path.join(__dirname, 'scroll-to-handle.test.html')}`;
    await page.goto(url);
    await sleep(1000);

    const selector = '#out-of-view-port';

    const previousClientRectangle = await getClientRectangleOf(selector, page);
    const previousViewportRectangle = await getViewportRectangleOf(page);

    const handle = await page.$(selector);
    const isSelectorVisibleBeforeScroll = await isHandleVisibleInViewport(
      handle,
      defaultVerboseOptions,
    );

    // When
    await SUT.scrollToHandle(handle);
    await sleep(2000);

    const currentClientRectangle = await getClientRectangleOf(selector, page);
    const currentViewportRectangle = await getViewportRectangleOf(page);
    const isSelectorVisibleAfterScroll = await isHandleVisibleInViewport(
      handle,
      defaultVerboseOptions,
    );

    // Then
    expect(isSelectorVisibleBeforeScroll).toBe(false);
    expect(isSelectorVisibleAfterScroll).toBe(true);
    expect(previousClientRectangle.top).toBeGreaterThan(currentClientRectangle.top);
    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
    expect(previousViewportRectangle!.pageTop).toBe(0);
    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
    expect(currentViewportRectangle!.pageTop).toBeGreaterThan(1000);
  });

  test('should not scroll to a hidden selector - firefox', async (): Promise<void> => {
    // Given
    browser = await firefox.launch({ headless: true });
    const browserContext = await browser.newContext({ viewport: null });
    const page = await browserContext.newPage();
    await showMousePosition(page);
    const url = `file:${path.join(__dirname, 'scroll-to-handle.test.html')}`;
    await page.goto(url);
    await sleep(1000);

    const selector = '#hidden';
    const handle = await page.$(selector);

    const previousClientRectangle = await getClientRectangleOf(selector, page);
    const previousViewportRectangle = await getViewportRectangleOf(page);

    // When
    await SUT.scrollToHandle(handle);
    await sleep(2000);

    const currentClientRectangle = await getClientRectangleOf(selector, page);
    const currentViewportRectangle = await getViewportRectangleOf(page);

    // Then
    expect(previousClientRectangle).toMatchObject(currentClientRectangle);
    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
    expect(previousViewportRectangle!.pageTop).toBe(0);
    expect(currentViewportRectangle).toMatchObject(previousViewportRectangle || {});
  });

  test('should scroll to a transparent selector - firefox', async (): Promise<void> => {
    // Given
    browser = await firefox.launch({ headless: true });
    const browserContext = await browser.newContext({ viewport: null });
    const page = await browserContext.newPage();
    await showMousePosition(page);
    const url = `file:${path.join(__dirname, 'scroll-to-handle.test.html')}`;
    await page.goto(url);
    await sleep(1000);

    const selector = '#transparent';
    const handle = await page.$(selector);

    const previousClientRectangle = await getClientRectangleOf(selector, page);
    const previousViewportRectangle = await getViewportRectangleOf(page);

    // When
    await SUT.scrollToHandle(handle);
    await sleep(2000);

    const currentClientRectangle = await getClientRectangleOf(selector, page);
    const currentViewportRectangle = await getViewportRectangleOf(page);

    // Then
    expect(previousClientRectangle.top).toBeGreaterThan(currentClientRectangle.top);
    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
    expect(previousViewportRectangle!.pageTop).toBe(0);
    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
    expect(currentViewportRectangle!.pageTop).toBeGreaterThan(1000);
  });
});
Example #16
Source File: main.ts    From actor-facebook-scraper with Apache License 2.0 4 votes vote down vote up
Apify.main(async () => {
    const input: Schema | null = await Apify.getInput() as any;

    if (!input || typeof input !== 'object') {
        throw new Error('Missing input');
    }

    const {
        startUrls = [],
        maxPosts = 3,
        maxPostDate,
        minPostDate,
        maxPostComments = 15,
        maxReviewDate,
        maxCommentDate,
        maxReviews = 3,
        commentsMode = 'RANKED_THREADED',
        scrapeAbout = false,
        countryCode = false,
        minCommentDate,
        scrapeReviews = true,
        scrapePosts = true,
        scrapeServices = true,
        language = 'en-US',
        sessionStorage = '',
        useStealth = false,
        debugLog = false,
        minPostComments,
        minPosts,
        maxConcurrency = 20,
        searchPages = [],
        searchLimit = 10,
        maxRequestRetries = 5,
    } = input;

    if (debugLog) {
        log.setLevel(log.LEVELS.DEBUG);
    }

    if ((!Array.isArray(startUrls) || !startUrls.length) && !searchPages?.length) {
        throw new Error('You must provide the "startUrls" input');
    }

    if (!Number.isFinite(maxPostComments)) {
        throw new Error('You must provide a finite number for "maxPostComments" input');
    }

    const proxyConfig = await proxyConfiguration({
        proxyConfig: input.proxyConfiguration,
        hint: ['RESIDENTIAL'],
        required: true,
    });

    const residentialWarning = () => {
        if (Apify.isAtHome() && !proxyConfig?.groups?.includes('RESIDENTIAL')) {
            log.warning(`!!!!!!!!!!!!!!!!!!!!!!!\n\nYou're not using RESIDENTIAL proxy group, it won't work as expected. Contact [email protected] or on Intercom to give you proxy trial\n\n!!!!!!!!!!!!!!!!!!!!!!!`);
        }
    };

    const shaderError = () => {
        if (Apify.isAtHome() && proxyConfig?.groups?.includes('SHADER')) {
            throw new Error(`Scraping Facebook with SHADER proxy group is not allowed! Please use RESIDENTIAL proxy group. Contact our support team through the Intercom widget that should appear on the bottom right corner of your screen for help.`);
        }
    };

    residentialWarning();
    shaderError();

    let handlePageTimeoutSecs = Math.round(60 * (((maxPostComments + maxPosts) || 10) * 0.08)) + 600; // minimum 600s

    if (handlePageTimeoutSecs * 60000 >= 0x7FFFFFFF) {
        log.warning(`maxPosts + maxPostComments parameter is too high, must be less than ${0x7FFFFFFF} milliseconds in total, got ${handlePageTimeoutSecs * 60000}. Loading posts and comments might never finish or crash the scraper at any moment.`, {
            maxPostComments,
            maxPosts,
            handlePageTimeoutSecs,
            handlePageTimeout: handlePageTimeoutSecs * 60000,
        });
        handlePageTimeoutSecs = Math.floor(0x7FFFFFFF / 60000);
    }

    log.info(`Will use ${handlePageTimeoutSecs}s timeout for page`);

    if (!(language in LANGUAGES)) {
        throw new Error(`Selected language "${language}" isn't supported`);
    }

    const { map, state, persistState } = await statePersistor();
    const elapsed = stopwatch();

    const postDate = minMaxDates({
        max: minPostDate,
        min: maxPostDate,
    });

    if (scrapePosts) {
        if (postDate.maxDate) {
            log.info(`\n-------\n\nGetting posts from ${postDate.maxDate.toLocaleString()} and older\n\n-------`);
        }

        if (postDate.minDate) {
            log.info(`\n-------\n\nGetting posts from ${postDate.minDate.toLocaleString()} and newer\n\n-------`);
        }
    }

    const commentDate = minMaxDates({
        min: maxCommentDate,
        max: minCommentDate,
    });

    if (commentDate.minDate) {
        log.info(`Getting comments from ${commentDate.minDate.toLocaleString()} and newer`);
    }

    const reviewDate = minMaxDates({
        min: maxReviewDate,
    });

    if (reviewDate.minDate) {
        log.info(`Getting reviews from ${reviewDate.minDate.toLocaleString()} and newer`);
    }

    const requestQueue = await Apify.openRequestQueue();

    if (!(startUrls?.length) && !(searchPages?.length)) {
        throw new Error('No requests were loaded from startUrls');
    }

    if (proxyConfig?.groups?.includes('RESIDENTIAL')) {
        proxyConfig.countryCode = countryCode ? language.split('-')?.[1] ?? 'US' : 'US';
    }

    log.info(`Using language "${(LANGUAGES as any)[language]}" (${language})`);

    const initSubPage = async (subpage: { url: string; section: FbSection, useMobile: boolean }, request: Apify.Request) => {
        if (subpage.section === 'home') {
            const username = extractUsernameFromUrl(subpage.url);

            // initialize the page. if it's already initialized,
            // use the current content
            await map.append(username, async (value) => {
                return {
                    ...emptyState(),
                    pageUrl: normalizeOutputPageUrl(subpage.url),
                    '#url': subpage.url,
                    '#ref': request.url,
                    ...value,
                };
            });
        }

        await requestQueue.addRequest({
            url: subpage.url,
            userData: {
                override: request.userData.override,
                label: LABELS.PAGE,
                sub: subpage.section,
                ref: request.url,
                useMobile: subpage.useMobile,
            },
        }, { forefront: true });
    };

    const pageInfo = [
        ...(scrapePosts ? ['posts'] : []),
        ...(scrapeReviews ? ['reviews'] : []),
        ...(scrapeServices ? ['services'] : []),
    ] as FbSection[];

    const addPageSearch = createAddPageSearch(requestQueue);

    for (const search of searchPages) {
        await addPageSearch(search);
    }

    let startUrlCount = 0;

    for await (const request of fromStartUrls(startUrls)) {
        try {
            let { url } = request;
            const urlType = getUrlLabel(url);

            if (urlType === LABELS.PAGE) {
                for (const subpage of generateSubpagesFromUrl(url, pageInfo)) {
                    await initSubPage(subpage, request);
                }
            } else if (urlType === LABELS.SEARCH) {
                await addPageSearch(url);
            } else if (urlType === LABELS.LISTING) {
                await requestQueue.addRequest({
                    url,
                    userData: {
                        override: request.userData.override,
                        label: urlType,
                        useMobile: false,
                    },
                });
            } else if (urlType === LABELS.POST || urlType === LABELS.PHOTO) {
                if (LABELS.PHOTO) {
                    url = photoToPost(url) ?? url;
                }

                const username = extractUsernameFromUrl(url);

                await requestQueue.addRequest({
                    url,
                    userData: {
                        override: request.userData.override,
                        label: LABELS.POST,
                        postId: fns.getPostId(url),
                        useMobile: false,
                        username,
                        canonical: storyFbToDesktopPermalink({ url, username })?.toString(),
                    },
                });

                // this is for home
                await initSubPage(generateSubpagesFromUrl(url, [])[0], request);
            }

            startUrlCount++;
        } catch (e) {
            if (e instanceof InfoError) {
                // We want to inform the rich error before throwing
                log.warning(`------\n\n${e.message}\n\n------`, e.toJSON());
            } else {
                throw e;
            }
        }
    }

    log.info(`Starting with ${startUrlCount} URLs`);

    const cache = resourceCache([
        /rsrc\.php/,
    ]);

    const extendOutputFunction = await extendFunction({
        map: async (data: Partial<FbPage>) => data,
        output: async (data) => {
            const finished = new Date().toISOString();

            data["#version"] = 4; // current data format version
            data['#finishedAt'] = finished;

            await Apify.pushData(data);
        },
        input,
        key: 'extendOutputFunction',
        helpers: {
            state,
            LABELS,
            fns,
            postDate,
            commentDate,
            reviewDate,
        },
    });

    const extendScraperFunction = await extendFunction({
        output: async () => {}, // eslint-disable-line @typescript-eslint/no-empty-function
        key: 'extendScraperFunction',
        input,
        helpers: {
            state,
            handlePageTimeoutSecs,
            cache,
            requestQueue,
            LABELS,
            addPageSearch,
            map,
            fns,
            postDate,
            commentDate,
            reviewDate,
        },
    });

    const fingerPrints = new FPG({
        browsers: [{ name: "firefox", minVersion: 90 }],
        locales: [language],
        operatingSystems: ['linux', 'android'],
    });

    const crawler = new Apify.PlaywrightCrawler({
        requestQueue,
        useSessionPool: true,
        sessionPoolOptions: {
            sessionOptions: {
                maxErrorScore: 0.5,
            },
        },
        maxRequestRetries,
        maxConcurrency,
        proxyConfiguration: proxyConfig,
        launchContext: {
            useIncognitoPages: true,
            launcher: firefox,
            launchOptions: {
                devtools: debugLog,
                headless: false,
            },
        },
        browserPoolOptions: {
            useFingerprints: false,
            retireBrowserAfterPageCount: 1,
            maxOpenPagesPerBrowser: 1, // required to use one IP per tab
            preLaunchHooks: [async (pageId, launchContext) => {
                const { request } = crawler.crawlingContexts.get(pageId);

                const { userData: { useMobile } } = request;

                const { fingerprint, headers } = fingerPrints.getFingerprint({
                    devices: useMobile ? ['mobile'] : ['desktop'],
                });

                request.userData.headers = headers;
                request.userData.fingerprint = fingerprint;

                launchContext.launchOptions = {
                    ...launchContext.launchOptions,
                    viewport: {
                        width: fingerprint.screen.width,
                        height: fingerprint.screen.height,
                    },
                    userAgent: fingerprint.userAgent,
                    locale: fingerprint.navigator.language,
                    bypassCSP: true,
                    ignoreHTTPSErrors: true,
                    fingerprint,
                };
            }],
            postPageCreateHooks: [async (page, browserController) => {
                const { launchContext } = browserController;
                const { useIncognitoPages, isFingerprintInjected, id } = launchContext;
                const { request } = crawler.crawlingContexts.get(id);

                if (isFingerprintInjected) {
                    // If not incognitoPages are used we would add the injection script over and over which could cause memory leaks.
                    return;
                }

                await new FingerprintInjector().attachFingerprintToPlaywright(page.context(), request.userData.fingerprint);

                if (!useIncognitoPages) {
                    // If not incognitoPages are used we would add the injection script over and over which could cause memory leaks.
                    launchContext.extend({ isFingerprintInjected: true });
                }
            }],
        },
        persistCookiesPerSession: false,
        handlePageTimeoutSecs, // more comments, less concurrency
        preNavigationHooks: [async ({ page, request, browserController }, gotoOptions) => {
            gotoOptions.waitUntil = request.userData.label === LABELS.POST
                || (request.userData.label === LABELS.PAGE && ['posts', 'reviews'].includes(request.userData.sub))
                ? 'load' : 'domcontentloaded';

            gotoOptions.timeout = 60000;

            await setLanguageCodeToCookie(language, page);

            await page.exposeFunction('unc', (element?: HTMLElement) => {
                try {
                    // weird bugs happen in this function, sometimes the dom element has no querySelectorAll for
                    // unknown reasons
                    if (!element) {
                        return;
                    }

                    element.className = '';
                    if (typeof element.removeAttribute === 'function') {
                        // weird bug that sometimes removeAttribute isn't a function?
                        element.removeAttribute('style');
                    }

                    if (typeof element.querySelectorAll === 'function') {
                        for (const el of [...element.querySelectorAll<HTMLElement>('*')]) {
                            el.className = ''; // removing the classes usually unhides

                            if (typeof element.removeAttribute === 'function') {
                                el.removeAttribute('style');
                            }
                        }
                    }
                } catch (e) {}
            });

            await cache(page);

            await page.addInitScript(() => {
                window.onerror = () => {};

                const f = () => {
                    for (const btn of document.querySelectorAll<HTMLButtonElement>('[data-testid="cookie-policy-dialog-accept-button"],[data-cookiebanner="accept_button"],#accept-cookie-banner-label')) {
                        if (btn) {
                            btn.click();
                        }
                    }
                    setTimeout(f, 1000);
                };
                setTimeout(f);
            });
        }],
        postNavigationHooks: [
            async ({ page }) => {
                if (!page.isClosed()) {
                    await page.bringToFront();
                }
            },
            async ({ page, request }) => {
                if (!page.isClosed()) {
                    // TODO: work around mixed context bug
                    if (page.url().includes(MOBILE_HOST) && !request.userData.useMobile) {
                        throw new InfoError(`Mismatched mobile / desktop`, {
                            namespace: 'internal',
                            url: request.url,
                        });
                    }
                }
            },
        ],
        handlePageFunction: async ({ request, page, session, response, browserController }) => {
            const { userData } = request;

            const label: FbLabel = userData.label; // eslint-disable-line prefer-destructuring

            log.debug(`Visiting page ${request.url}`, userData);

            try {
                if (page.url().includes('?next=')) {
                    throw new InfoError(`Content redirected to login page, retrying...`, {
                        url: request.url,
                        namespace: 'login',
                        userData,
                    });
                }

                if (userData.useMobile) {
                    // need to do some checks if the current mobile page is the interactive one or if
                    // it has been blocked
                    if (await page.$(CSS_SELECTORS.MOBILE_CAPTCHA)) {
                        throw new InfoError('Mobile captcha found', {
                            url: request.url,
                            namespace: 'captcha',
                            userData,
                        });
                    }

                    try {
                        await Promise.all([
                            page.waitForSelector(CSS_SELECTORS.MOBILE_META, {
                                timeout: 15000, // sometimes the page takes a while to load the responsive interactive version,
                                state: 'attached',
                            }),
                            page.waitForSelector(CSS_SELECTORS.MOBILE_BODY_CLASS, {
                                timeout: 15000, // correctly detected android. if this isn't the case, the image names will change
                                state: 'attached',
                            }),
                        ]);
                    } catch (e) {
                        throw new InfoError(`An unexpected page layout was returned by the server. This request will be retried shortly.${e.message}`, {
                            url: request.url,
                            namespace: 'mobile-meta',
                            userData,
                        });
                    }
                }

                if (!userData.useMobile && await page.$(CSS_SELECTORS.DESKTOP_CAPTCHA)) {
                    throw new InfoError('Desktop captcha found', {
                        url: request.url,
                        namespace: 'captcha',
                        userData,
                    });
                }

                if (await page.$eval('title', (el) => el.textContent === 'Error') || response.statusCode === 500) {
                    throw new InfoError('Facebook internal error, maybe it\'s going through instability, it will be retried', {
                        url: request.url,
                        namespace: 'internal',
                        userData,
                    });
                }

                if (label !== LABELS.LISTING
                    && label !== LABELS.SEARCH
                    && label !== LABELS.POST
                    && request.userData.sub !== 'posts'
                    && await isNotFoundPage(page)) {
                    request.noRetry = true;

                    // throw away if page is not available
                    // but inform the user of error
                    throw new InfoError('Content not found. This either means the page doesn\'t exist, or the section itself doesn\'t exist (about, reviews, services)', {
                        url: request.url,
                        namespace: 'isNotFoundPage',
                        userData,
                    });
                }

                await page.evaluate(() => {
                    window.onerror = () => {};
                });

                if (label === LABELS.LISTING) {
                    const start = stopwatch();
                    const pagesUrls = await getPagesFromListing(page);

                    for (const url of pagesUrls) {
                        for (const subpage of generateSubpagesFromUrl(url, pageInfo)) {
                            await initSubPage(subpage, request);
                        }
                    }

                    log.info(`Got ${pagesUrls.size} pages from listing in ${start() / 1000}s`);
                } else if (userData.label === LABELS.SEARCH) {
                    const start = stopwatch();
                    let count = 0;

                    for await (const url of getPagesFromSearch(page, searchLimit)) {
                        count++;

                        for (const subpage of generateSubpagesFromUrl(url, pageInfo)) {
                            await initSubPage(subpage, request);
                        }
                    }

                    log.info(`Got ${count} pages from search "${userData.searchTerm}" in ${start() / 1000}s`);
                } else if (userData.label === LABELS.PAGE) {
                    const username = extractUsernameFromUrl(request.url);

                    switch (userData.sub) {
                        // Main landing page
                        case 'home':
                            await map.append(username, async (value) => {
                                const {
                                    likes,
                                    messenger,
                                    title,
                                    verified,
                                    ...address
                                } = await getPageInfo(page);

                                return getFieldInfos(page, {
                                    ...value,
                                    likes,
                                    messenger,
                                    title,
                                    verified,
                                    address: {
                                        lat: null,
                                        lng: null,
                                        ...value?.address,
                                        ...address,
                                    },
                                });
                            });
                            break;
                        // Services if any
                        case 'services':
                            try {
                                const services = await getServices(page);

                                if (services.length) {
                                    await map.append(username, async (value) => {
                                        return {
                                            ...value,
                                            services: [
                                                ...(value?.services ?? []),
                                                ...services,
                                            ],
                                        };
                                    });
                                }
                            } catch (e) {
                                // it's ok to fail here, not every page has services
                                log.debug(e.message);
                            }
                            break;
                        // About if any
                        case 'about':
                            await map.append(username, async (value) => {
                                return getFieldInfos(page, {
                                    ...value,
                                });
                            });
                            break;
                        // Posts
                        case 'posts': {
                            let max = maxPosts;
                            let date = postDate;

                            const { overriden, settings } = overrideUserData(input, request);

                            if (overriden) {
                                if (settings?.maxPosts) {
                                    max = settings.maxPosts;
                                }

                                if (settings?.maxPostDate || settings?.minPostDate) {
                                    date = minMaxDates({
                                        min: settings.maxPostDate,
                                        max: settings.minPostDate,
                                    });
                                }
                            }

                            // We don't do anything here, we enqueue posts to be
                            // read on their own phase/label
                            const postCount = await getPostUrls(page, {
                                max,
                                date,
                                username,
                                requestQueue,
                                request,
                                minPosts,
                            });

                            if (maxPosts && minPosts && postCount < minPosts) {
                                throw new InfoError(`Minimum post count of ${minPosts} not met, retrying...`, {
                                    namespace: 'threshold',
                                    url: page.url(),
                                });
                            }

                            break;
                        }
                        // Reviews if any
                        case 'reviews':
                            try {
                                const reviewsData = await getReviews(page, {
                                    max: maxReviews,
                                    date: reviewDate,
                                    request,
                                });

                                if (reviewsData) {
                                    const { average, count, reviews } = reviewsData;

                                    await map.append(username, async (value) => {
                                        return {
                                            ...value,
                                            reviews: {
                                                ...(value?.reviews ?? {}),
                                                average,
                                                count,
                                                reviews: [
                                                    ...reviews,
                                                    ...(value?.reviews?.reviews ?? []),
                                                ],
                                            },
                                        };
                                    });
                                }
                            } catch (e) {
                                // it's ok for failing here, not every page has reviews
                                log.debug(e.message);
                            }
                            break;
                        // make eslint happy
                        default:
                    }
                } else if (label === LABELS.POST) {
                    const postTimer = stopwatch();

                    log.debug('Started processing post', { url: request.url, postId: request.userData.postId });

                    // actually parse post content here, it doesn't work on
                    // mobile address
                    const { username } = userData;

                    const [postStats, content] = await Promise.all([
                        getPostInfoFromScript(page, request),
                        getPostContent(page),
                    ]);

                    const { overriden, settings } = overrideUserData(input, request);
                    let mode: FbCommentsMode = commentsMode;
                    let date: typeof commentDate = commentDate;
                    let max = maxPostComments;
                    let minComments = minPostComments;

                    if (overriden) {
                        if (settings?.minCommentDate || settings?.maxCommentDate) {
                            date = minMaxDates({
                                max: settings.minCommentDate,
                                min: settings.maxCommentDate,
                            });
                        }

                        if (settings?.maxPostComments) {
                            max = settings.maxPostComments;
                        }

                        if (settings?.commentsMode) {
                            mode = settings.commentsMode;
                        }

                        if (settings?.minPostComments) {
                            minComments = settings.minPostComments;
                        }
                    }

                    const existingPost = await map.read(username).then((p) => p?.posts?.find((post) => post.postUrl === content.postUrl));
                    const postContent: FbPost = existingPost || {
                        ...content as FbPost,
                        postStats,
                        postComments: {
                            count: 0,
                            mode,
                            comments: [],
                        },
                    };

                    if (!existingPost) {
                        await map.append(username, async (value) => {
                            return {
                                ...value,
                                posts: [
                                    postContent,
                                    ...(value?.posts ?? []),
                                ],
                            } as Partial<FbPage>;
                        });
                    }

                    if (postStats.comments > 0) {
                        const postCount = await getPostComments(page, {
                            max,
                            mode,
                            date,
                            request,
                            add: async (comment) => {
                                await map.append(username, async (value) => {
                                    postContent.postComments.comments.push(comment);
                                    return value;
                                });
                            },
                        });

                        await map.append(username, async (value) => {
                            postContent.postComments.count = postCount;
                            return value;
                        });

                        if (max && minComments && (postContent?.postComments?.comments?.length ?? 0) < minComments) {
                            throw new InfoError(`Minimum comment count ${minComments} not met, retrying`, {
                                namespace: 'threshold',
                                url: page.url(),
                            });
                        }
                    }

                    log.info(`Processed post in ${postTimer() / 1000}s`, { url: request.url });
                }
            } catch (e) {
                log.debug(e.message, {
                    url: request.url,
                    userData: request.userData,
                    error: e,
                });

                if (e instanceof InfoError) {
                    // We want to inform the rich error before throwing
                    log.warning(e.message, e.toJSON());

                    if (['captcha', 'mobile-meta', 'getFieldInfos', 'internal', 'login', 'threshold'].includes(e.meta.namespace)) {
                        session.retire();
                        await browserController.close(page);
                    }
                }

                throw e;
            } finally {
                await extendScraperFunction(undefined, {
                    page,
                    request,
                    session,
                    username: extractUsernameFromUrl(request.url),
                    label: 'HANDLE',
                });
            }

            log.debug(`Done with page ${request.url}`);
        },
        handleFailedRequestFunction: async ({ request, error }) => {
            // this only happens when maxRetries is
            // comprised mainly of InfoError, which is usually a problem
            // with pages
            log.exception(error, `The request failed after all ${request.retryCount} retries, last error was:`, error instanceof InfoError
                ? error.toJSON()
                : {});
        },
    });

    await extendScraperFunction(undefined, {
        label: 'SETUP',
        crawler,
    });

    if (!debugLog) {
        fns.patchLog(crawler);
    }

    await crawler.run();

    await extendScraperFunction(undefined, {
        label: 'FINISH',
        crawler,
    });

    await persistState();

    log.info('Generating dataset...');

    // generate the dataset from all the crawled pages
    for (const page of state.values()) {
        await extendOutputFunction(page, {});
    }

    residentialWarning();

    log.info(`Done in ${Math.round(elapsed() / 60000)}m!`);
});