puppeteer#ElementHandle TypeScript Examples

launch = async ({
  path = '/',
  disableJavascript = false,
}: LaunchParams = {}): Promise<ElementHandle> => {
  if (disableJavascript) {
    await page.setJavaScriptEnabled(false);
  }

  await page.goto(
    url(path),
    disableJavascript ? {} : { waitUntil: 'domcontentloaded' },
  );

  if (clientOnly) {
    await page.waitFor(500); // gatsby develop takes a moment to warm up on first load
  }

  return getDocument(page as Page);
}

assertTags = async (
  assertions: TagAssertionBuilder[],
  $document: ElementHandle,
) => {
  for (const assertion of assertions) {
    const { prop: p, result, selector, indexes } = assertion;
    if (Array.isArray(result)) {
      await props($document.$$(selector), p).then(async (content) => {
        if (Array.isArray(indexes)) {
          expect(indexes.length).toBe(result.length); // Ensure the indexes match;
          indexes.forEach((ii, index) => {
            expect(content[ii]).toBe(result[index]);
          });
        } else {
          expect(content).toHaveLength(result.length);
          expect(content).toEqual(result);
        }
      });
    } else {
      await expect(prop($document.$(selector), p)).resolves.toBe(result);
    }
  }
}

private async login(page: Page): Promise<void> {
    this.L.trace('Navigating to Epic Games login page');
    await Promise.all([
      page.goto(
        `https://www.epicgames.com/id/login/epic?redirect_uri=${STORE_HOMEPAGE_EN}&client_id=${EPIC_CLIENT_ID}`
      ),
      page.waitForNavigation({ waitUntil: 'networkidle0' }),
    ]);
    this.L.trace('Waiting for email field');
    const emailElem = (await page.waitForSelector('#email')) as ElementHandle<HTMLInputElement>;
    this.L.trace('Filling email field');
    await emailElem.type(this.email);
    this.L.trace('Waiting for password field');
    const passElem = (await page.waitForSelector('#password')) as ElementHandle<HTMLInputElement>;
    this.L.trace('Filling password field');
    await passElem.type(this.password);
    this.L.trace('Waiting for sign-in button');
    const [signInElem] = await Promise.all([
      page.waitForSelector('#sign-in:not([disabled])') as Promise<ElementHandle<HTMLInputElement>>,
      // page.waitForNetworkIdle(),
    ]);
    // Remember me should be checked by default
    this.L.trace('Clicking sign-in button');
    await signInElem.hover();
    await signInElem.focus();
    await Promise.all([await signInElem.click({ delay: 100 }), await this.handleLoginClick(page)]);
  }

private async handleMfa(page: Page, input: ElementHandle<HTMLInputElement>): Promise<void> {
    this.L.trace('MFA detected');
    if (!this.totp) throw new Error('TOTP required for MFA login');
    const totp = new TOTP({ secret: this.totp });
    const mfaCode = totp.generate();
    this.L.trace('Filling MFA field');
    await input.type(mfaCode);
    this.L.trace('Waiting for continue button');
    const continueButton = (await page.waitForSelector(
      `button#continue`
    )) as ElementHandle<HTMLButtonElement>;
    this.L.trace('Clicking continue button');
    await Promise.all([
      await continueButton.click({ delay: 100 }),
      await page.waitForNavigation({ waitUntil: 'networkidle0' }),
    ]);
  }

prop = async <GReturn>(
  $element: ElementHandle | Promise<ElementHandle>,
  property: string,
): Promise<GReturn> => {
  try {
    const handle = await $element;
    const propertyHandle = await handle.getProperty(property);
    return propertyHandle.jsonValue() as any;
  } catch (e) {
    console.error(property, $element.toString());
    throw e;
  }
}

private async handleLoginClick(page: Page): Promise<void> {
    this.L.trace('Waiting for sign-in result');
    const result = await Promise.race([
      this.waitForHCaptcha(page),
      this.waitForRedirectNav(page),
      this.waitForError(page),
      this.waitForMfaInput(page),
    ]);
    if (result === 'captcha') {
      this.L.trace('Captcha detected');
      await this.openPortalAndNotify(page, NotificationReason.LOGIN);
      await this.handleCaptchaSolved(page);
      await page.closePortal();
    } else if (result === 'nav') {
      this.L.debug('Redirected to store page, login successful');
    } else if (typeof result === 'string') {
      this.L.warn(`Login returned error: ${result}`);
      await this.login(page);
    } else {
      await this.handleMfa(page, result as ElementHandle<HTMLInputElement>);
    }
  }

private async handleCaptchaSolved(page: Page): Promise<void> {
    this.L.trace('Waiting for MFA possibility');
    const result = await Promise.race([
      this.waitForMfaInput(page),
      this.waitForRedirectNav(page),
      this.waitForError(page),
    ]);
    if (typeof result !== 'string') {
      // result is an ElementHandle
      await this.handleMfa(page, result as ElementHandle<HTMLInputElement>);
    } else if (result !== 'nav') {
      // result is an error message
      this.L.warn(`Login returned error: ${result}`);
      await this.login(page);
    }
    // result is 'nav', success
  }

private async fillDOB(page: Page): Promise<void> {
    this.L.trace('Getting date fields');
    const [monthInput, dayInput, yearInput] = await Promise.all([
      page.waitForSelector(`#month`) as Promise<ElementHandle<HTMLDivElement>>,
      page.waitForSelector(`#day`) as Promise<ElementHandle<HTMLDivElement>>,
      page.waitForSelector(`#year`) as Promise<ElementHandle<HTMLInputElement>>,
    ]);
    await monthInput.click();
    const month1 = (await page.waitForSelector(
      `ul.MuiList-root > li`
    )) as ElementHandle<HTMLLIElement>;
    await month1.click();
    await page.waitForTimeout(500); // idk why this is required
    await dayInput.click();
    const day1 = (await page.waitForSelector(
      `ul.MuiList-root > li`
    )) as ElementHandle<HTMLLIElement>;
    await day1.click();
    await yearInput.type(this.getRandomInt(1970, 2002).toString());
    const continueButton = (await page.waitForSelector(
      `#continue:not([disabled])`
    )) as ElementHandle<HTMLButtonElement>;
    await page.waitForTimeout(500); // idk why this is required
    this.L.trace('Clicking continueButton');
    await continueButton.click({ delay: 100 });
  }

private async fillEmailVerificationForm(page: Page): Promise<void> {
    this.L.trace('Working on email verification form');
    const code = await this.getVerification();
    this.L.trace('Waiting for codeInput');
    const codeInput = (await page.waitForSelector(
      `input[name='code-input-0']`
    )) as ElementHandle<HTMLInputElement>;
    await codeInput.click({ delay: 100 });
    await page.keyboard.type(code);
    this.L.trace('Waiting for continueButton');
    const continueButton = (await page.waitForSelector(
      `#continue:not([disabled])`
    )) as ElementHandle<HTMLButtonElement>;
    this.L.trace('Clicking continueButton');
    await continueButton.click();
    await page.waitForNavigation({ waitUntil: 'networkidle2' });
  }

private async fillSignUpForm(page: Page): Promise<void> {
    this.L.trace('Getting sign up fields');
    const randName = new RandExp(/[a-zA-Z]{3,12}/);
    const [
      countryInput,
      firstNameInput,
      lastNameInput,
      displayNameInput,
      emailInput,
      passwordInput,
      tosInput,
    ] = await Promise.all([
      page.waitForSelector(`#country`) as Promise<ElementHandle<Element>>,
      page.waitForSelector(`#name`) as Promise<ElementHandle<HTMLInputElement>>,
      page.waitForSelector(`#lastName`) as Promise<ElementHandle<HTMLInputElement>>,
      page.waitForSelector(`#displayName`) as Promise<ElementHandle<HTMLInputElement>>,
      page.waitForSelector(`#email`) as Promise<ElementHandle<HTMLInputElement>>,
      page.waitForSelector(`#password`) as Promise<ElementHandle<HTMLInputElement>>,
      page.waitForSelector(`#tos`) as Promise<ElementHandle<HTMLInputElement>>,
    ]);
    await countryInput.type(this.country);
    await firstNameInput.type(randName.gen());
    await lastNameInput.type(randName.gen());
    await displayNameInput.type(this.username);
    await emailInput.type(this.email);
    await passwordInput.type(this.password);
    await tosInput.click();
    const submitButton = (await page.waitForSelector(
      `#btn-submit:not([disabled])`
    )) as ElementHandle<HTMLButtonElement>;
    this.L.trace('Clicking submitButton');
    await submitButton.click({ delay: 100 });
  }

root = async () => await page.$(rootSelector) as ElementHandle

props = async <GReturn>(
  $elements: ElementHandle[] | Promise<ElementHandle[]>,
  property: string,
): Promise<GReturn[]> => {
  const handles = await $elements;
  return Promise.all(handles.map((handle) => prop<GReturn>(handle, property)));
}

$document: ElementHandle

public async deleteAccount(): Promise<void> {
    this.L.info({ email: this.email }, 'Deleting account');

    const hCaptchaCookies = await getHcaptchaCookies();
    const userCookies = await getCookiesRaw(this.email);
    const puppeteerCookies = toughCookieFileStoreToPuppeteerCookie(userCookies);
    this.L.debug('Logging in with puppeteer');
    const browser = await puppeteer.launch(launchArgs);
    const page = await browser.newPage();
    this.L.trace(getDevtoolsUrl(page));
    const cdpClient = await page.target().createCDPSession();
    await cdpClient.send('Network.setCookies', {
      cookies: [...puppeteerCookies, ...hCaptchaCookies],
    });
    await page.setCookie(...puppeteerCookies, ...hCaptchaCookies);
    await page.goto(`https://www.epicgames.com/account/personal`, { waitUntil: 'networkidle0' });
    this.L.trace('Waiting for deleteButton');
    const deleteButton = (await page.waitForXPath(
      `//button[contains(., 'Request Account Delete')]`
    )) as ElementHandle<HTMLButtonElement>;
    this.L.trace('Clicking deleteButton');
    await deleteButton.click({ delay: 100 });
    this.L.trace('Waiting for securityCodeInput');
    const securityCodeInput = (await page.waitForSelector(
      `input[name='security-code']`
    )) as ElementHandle<HTMLInputElement>;
    const code = await this.getActionVerification();
    this.L.trace('Filling securityCodeInput');
    await securityCodeInput.type(code);
    this.L.trace('Waiting for confirmButton');
    const confirmButton = (await page.waitForXPath(
      `//button[contains(., 'Confirm Delete Request')]`
    )) as ElementHandle<HTMLButtonElement>;
    this.L.trace('Clicking confirmButton');
    await confirmButton.click({ delay: 100 });
    this.L.trace('Waiting for skipSurveyButton');
    const skipSurveyButton = (await page.waitForSelector(
      `button#deletion-reason-skip`
    )) as ElementHandle<HTMLButtonElement>;
    this.L.trace('Clicking skipSurveyButton');
    await skipSurveyButton.click({ delay: 100 });
    await page.waitForSelector(`div.account-deletion-request-success-modal`);
    this.L.debug('Account deletion successful');

    this.L.trace('Saving new cookies');
    const currentUrlCookies = (await cdpClient.send('Network.getAllCookies')) as {
      cookies: Protocol.Network.Cookie[];
    };
    await browser.close();
    setPuppeteerCookies(this.email, currentUrlCookies.cookies);
  }

private async waitForMfaInput(
    page: Page,
    timeout = NOTIFICATION_TIMEOUT
  ): Promise<ElementHandle<HTMLInputElement>> {
    return page.waitForSelector(`input[name="code-input-0"]`, {
      timeout,
    }) as Promise<ElementHandle<HTMLInputElement>>;
  }

private async waitForError(page: Page, timeout = NOTIFICATION_TIMEOUT): Promise<string> {
    const errorHeader = (await page.waitForSelector('div[role="alert"] > h6:first-of-type', {
      timeout,
      visible: true,
    })) as ElementHandle<HTMLHeadingElement>;
    return errorHeader.evaluate((el) => el.innerText);
  }

getHcaptchaCookies = async (): Promise<Protocol.Network.Cookie[]> => {
  const { hcaptchaAccessibilityUrl } = config;
  if (!hcaptchaAccessibilityUrl) {
    L.debug(
      'hcaptchaAccessibilityUrl not configured, captchas are less likely to be bypassed. Follow this guide to set it up: https://github.com/claabs/epicgames-freegames-node#hcaptcha-accessibility-cookies'
    );
    return [];
  }
  let cookieData = await getCookieCache();
  let browser;
  if (!cookieData) {
    try {
      L.debug('Setting hCaptcha accessibility cookies');
      browser = await safeLaunchBrowser(L);
      const page = await safeNewPage(browser, L);

      L.trace(getDevtoolsUrl(page));
      L.trace(`Navigating to ${hcaptchaAccessibilityUrl}`);
      await Promise.all([
        page.goto(hcaptchaAccessibilityUrl),
        page.waitForNavigation({ waitUntil: 'networkidle0' }),
      ]);
      L.trace(`Waiting for setAccessibilityCookie button`);
      const setCookieButton = (await page.waitForSelector(
        `button[data-cy='setAccessibilityCookie']:not([disabled])`
      )) as ElementHandle<HTMLButtonElement>;
      L.trace(`Clicking setAccessibilityCookie button`);
      await setCookieButton.click({ delay: 100 });
      try {
        const getCookieResp = await page.waitForResponse(
          (res) =>
            res.url() === 'https://accounts.hcaptcha.com/accessibility/get_cookie' &&
            res.request().method() === 'POST'
        );
        const getCookieStatus = getCookieResp.status();
        if (getCookieStatus !== 200) {
          const errorBody = await getCookieResp.json();
          L.debug(
            { status: getCookieStatus, errorBody },
            'Error from hCaptcha get_cookie request, continuing without hCaptcha accessibility cookies'
          );
        }
      } catch (err) {
        L.debug(err);
        L.warn(
          'No get cookie response recieved, continuing without hCaptcha accessibility cookies'
        );
      }
      L.trace(`Saving new cookies`);
      const cdpClient = await page.target().createCDPSession();
      const currentUrlCookies = (await cdpClient.send('Network.getAllCookies')) as {
        cookies: Protocol.Network.Cookie[];
      };
      await browser.close();
      cookieData = currentUrlCookies.cookies;
      await setCookieCache(cookieData);
    } catch (err) {
      L.warn(err);
      L.warn(
        'Setting the hCaptcha accessibility cookies encountered an error. Continuing without them...'
      );
      if (browser) await browser.close();
      return [];
    }
  }
  return cookieData;
}

/**
   * Extract data from a group post
   * @param post
   */
  public async parsePost(post: ElementHandle) {
    if (this.page === undefined || this.config === undefined) {
      throw new InitialisationError();
    }

    const { authorName, content } = await this.page.evaluate(
      async (postElm: HTMLElement, cssSelectors: typeof selectors): Promise<any> => {
        let postAuthorElm;
        postAuthorElm = <HTMLElement>postElm.querySelector(
          cssSelectors.facebook_post.post_author,
        );
        let postAuthorName;
        // Not all posts provide author profile url
        if (postAuthorElm) {
          postAuthorName = postAuthorElm.innerText;
        } else {
          postAuthorElm = <HTMLElement>postElm.querySelector(
            cssSelectors.facebook_post.post_author2,
          );
          postAuthorName = postAuthorElm.innerText;
        }

        const postContentElm = <HTMLElement>postElm.querySelector(
          cssSelectors.facebook_post.post_content,
        );
        let postContent;
        // Some posts don't have text, so they won't have postContentElm
        if (postContentElm) {
          // We should click the "See More..." button before extracting the post content
          const expandButton = <HTMLElement>postContentElm.querySelector(
            cssSelectors.facebook_post.post_content_expand_button,
          );
          if (expandButton) {
            postContent = await new Promise((res) => {
              const observer = new MutationObserver(
                () => {
                  observer.disconnect();
                  res(postContentElm.innerText);
                },
              );
              observer.observe(postContentElm, { childList: true, subtree: true });
              expandButton.click();
            });
          } else {
            postContent = postContentElm.innerText;
          }
        } else {
          postContent = '';
        }

        return {
          authorName: postAuthorName,
          content: postContent,
        };
      },
      post, selectors,
    );

    // crates a submission object which contains our submission
    const submission: GroupPost = {
      author: authorName,
      post: content,
    };

    return submission;
  }

/**
   * Completes a purchase starting from the purchase iframe using its namespace and offerId
   */
  public async purchaseShort(namespace: string, offer: string): Promise<void> {
    const page = await this.setupPage();
    try {
      const purchaseUrl = `${EPIC_PURCHASE_ENDPOINT}?highlightColor=0078f2&offers=1-${namespace}-${offer}&orderId&purchaseToken&showNavigation=true`;

      /**
       * This inner function is declared to allow the page to be refreshed after a 3 hour timeout
       */
      // eslint-disable-next-line consistent-return
      const initPurchase = async (): Promise<void> => {
        this.L.info({ purchaseUrl }, 'Loading purchase page');
        await page.goto(purchaseUrl, { waitUntil: 'networkidle0' });
        await page.waitForNetworkIdle({ idleTime: 2000 });
        try {
          this.L.trace('Waiting for cookieDialog');
          const cookieDialog = (await page.waitForSelector(`button#onetrust-accept-btn-handler`, {
            timeout: 3000,
          })) as ElementHandle<HTMLButtonElement>;
          this.L.trace('Clicking cookieDialog');
          await cookieDialog.click({ delay: 100 });
        } catch (err) {
          if (!err.message.includes('timeout')) {
            throw err;
          }
          this.L.trace('No cookie dialog presented');
        }
        this.L.trace('Waiting for placeOrderButton');
        const placeOrderButton = (await page.waitForSelector(
          `button.payment-btn:not([disabled])`
        )) as ElementHandle<HTMLButtonElement>;
        this.L.debug('Clicking placeOrderButton');
        await placeOrderButton.click({ delay: 100 });
        try {
          const euRefundAgreeButton = (await page.waitForSelector(
            `div.payment-confirm__actions > button.payment-btn.payment-confirm__btn.payment-btn--primary`,
            { timeout: 3000 }
          )) as ElementHandle<HTMLButtonElement>;
          this.L.debug('Clicking euRefundAgreeButton');
          await euRefundAgreeButton.click({ delay: 100 });
        } catch (err) {
          if (!err.message.includes('timeout')) {
            throw err;
          }
          this.L.trace('No EU "Refund and Right of Withdrawal Information" dialog presented');
        }
        this.L.debug('Waiting for receipt');
        const purchaseEvent = await Promise.race([
          page
            .waitForFunction(() => document.location.hash.includes('/purchase/receipt'))
            .then(() => 'nav'),
          page
            .waitForSelector('.payment-alert--ERROR > span.payment-alert__content')
            .then((errorHandle: ElementHandle<HTMLSpanElement> | null) =>
              errorHandle ? errorHandle.evaluate((el) => el.innerText) : 'Unknown purchase error'
            ),
          this.waitForHCaptcha(page),
        ]);
        if (purchaseEvent === 'captcha') {
          this.L.debug('Captcha detected');
          // Keep the existing portal open
          if (!page.hasOpenPortal()) {
            await this.openPortalAndNotify(page, NotificationReason.PURCHASE);
          }
          const interactionResult = await Promise.race([
            page
              .waitForFunction(() => document.location.hash.includes('/purchase/receipt'), {
                timeout: NOTIFICATION_TIMEOUT,
              })
              .then(() => 'nav'),
            page.waitForTimeout(3 * 60 * 60 * 1000).then(() => 'timeout'),
          ]);
          if (interactionResult === 'timeout') {
            this.L.info('Reloading purchase page...'); // Reload page after 3 hour timeout
            return initPurchase();
          }
          await page.closePortal();
        } else if (purchaseEvent !== 'nav') {
          throw new Error(purchaseEvent);
        }
      };

      await initPurchase();
      this.L.trace(`Puppeteer purchase successful`);
      await this.teardownPage(page);
    } catch (err) {
      let success = false;
      if (page) {
        const errorPrefix = `error-${new Date().toISOString()}`.replace(/:/g, '-');
        const errorImage = path.join(CONFIG_DIR, `${errorPrefix}.png`);
        await page.screenshot({ path: errorImage });
        const errorHtml = path.join(CONFIG_DIR, `${errorPrefix}.html`);
        const htmlContent = await page.content();
        outputFileSync(errorHtml, htmlContent, 'utf8');
        this.L.warn(err);
        this.L.error(
          { errorImage, errorHtml },
          'Encountered an error during browser automation. Saved a screenshot and page HTML for debugging purposes.'
        );
        if (!config.noHumanErrorHelp) success = await this.sendErrorManualHelpNotification(page);
        await page.close();
      }
      if (!success) throw err;
    }
  }

/**
   * Function saves the group posts for the given groupId
   * @param groupId
   * @param outputFileName
   */
  public async getGroupPosts(
    groupId: number,
    outputFileName: string | undefined,
    callback?: (arg0: GroupPost) => void,
    save: boolean = true,
  ) {
    if (this.page === undefined || this.config === undefined) {
      throw new InitialisationError();
    }
    const groupUrl = generateFacebookGroupURLById(groupId);
    await this.page.goto(
      groupUrl,
      {
        timeout: 600000,
        waitUntil: 'domcontentloaded',
      },
    );

    if (outputFileName === undefined) {
      // eslint-disable-next-line no-param-reassign
      outputFileName = `${this.config.output + groupId}.json`;
    }

    /**
     * Save post to the database
     * @param postData
     */
    const savePost = (postData: GroupPost): void => {
      const allPublicationsList = getOldPublications(outputFileName!);
      allPublicationsList.push(postData);
      if (save) {
        fs.writeFileSync(
          outputFileName!,
          JSON.stringify(allPublicationsList, undefined, 4),
          { encoding: 'utf8' },
        );
      }
    };

    // Start Scrolling!
    this.page.evaluate(autoScroll);

    /**
     * Waiting for the group feed container to continue
     * and to avoid the selector not found error.
     * Note that we ignore any posts outside this container
     * specifically announcements, because they don't follow
     * the same sorting method as the others.
     * */
    await this.page.waitForSelector(
      selectors.facebook_group.group_feed_container,
    );

    /**
     * Handle new added posts
     */
    const handlePosts = async () => {
      const post = await this.page?.evaluateHandle(
        () => window.posts.shift(),
      );
      const postData = await this.parsePost(<ElementHandle>post);
      if (callback !== undefined && callback !== null) {
        callback(postData);
      }
      savePost(postData);
    };
    this.page.exposeFunction('handlePosts', handlePosts);

    // Listen to new added posts
    this.page.evaluate((cssSelectors: typeof selectors) => {
      window.posts = [];
      const target = <HTMLElement>document.querySelector(
        cssSelectors.facebook_group.group_feed_container,
      );
      const observer = new MutationObserver((mutations) => {
        for (let i = 0; i < mutations.length; i += 1) {
          for (let j = 0; j < mutations[i].addedNodes.length; j += 1) {
            const addedNode = <HTMLElement>mutations[i].addedNodes[j];
            const postElm = <HTMLElement>addedNode.querySelector(
              cssSelectors.facebook_post.post_element,
            );
            if (postElm) {
              window.posts.push(postElm);
              handlePosts();
            }
          }
        }
      });
      observer.observe(target, { childList: true });
    }, selectors);
  }