puppeteer HTTPResponse TypeScript Examples

Source File: platform.ts From cli with Apache License 2.0

6 votes

export function isSearchRequestOrResponse(
  requestOrResponse: HTTPRequest | HTTPResponse
) {
  const searchUrl = new URL(
    '/rest/search/v2?organizationId',
    process.env.PLATFORM_HOST
  );
  return requestOrResponse.url().startsWith(searchUrl.href);
}

Source File: solver.ts From FlareSolverr with MIT License

5 votes

async function gotoPage(params: V1Request, page: Page): Promise<HTTPResponse> {
    let pageTimeout = params.maxTimeout / 3;
    let response: HTTPResponse
    try {
        response = await page.goto(params.url, {waitUntil: 'domcontentloaded', timeout: pageTimeout});
    } catch (e) {
        // retry
        response = await page.goto(params.url, {waitUntil: 'domcontentloaded', timeout: pageTimeout});
    }

    if (params.method == 'POST') {
        // post hack
        await page.setContent(
            `
<!DOCTYPE html>
<html>
<body>
<script>

  function parseQuery(queryString) {
    var query = {};
    var pairs = (queryString[0] === '?' ? queryString.substr(1) : queryString).split('&');
    for (var i = 0; i < pairs.length; i++) {
        var pair = pairs[i].split('=');
        query[decodeURIComponent(pair[0])] = decodeURIComponent(pair[1] || '');
    }
    return query;
  }

  const form = document.createElement('form');
  form.method = 'POST';
  form.action = '${params.url}';

  const params = parseQuery('${params.postData}');
  for (const key in params) {
    if (params.hasOwnProperty(key)) {
      const hiddenField = document.createElement('input');
      hiddenField.type = 'hidden';
      hiddenField.name = key;
      hiddenField.value = params[key];
      form.appendChild(hiddenField);
    }
  }

  document.body.appendChild(form);
  form.submit();
    
</script>
</body>
</html> 
            `
        );
        await page.waitForTimeout(2000)
        try {
            await page.waitForNavigation({waitUntil: 'domcontentloaded', timeout: 2000})
        } catch (e) {}

    }
    return response
}

Source File: cloudflare.ts From FlareSolverr with MIT License

4 votes

export default async function resolveChallenge(url: string, page: Page, response: HTTPResponse): Promise<HTTPResponse> {

  // look for challenge and return fast if not detected
  let cfDetected = response.headers().server &&
      (response.headers().server.startsWith('cloudflare') || response.headers().server.startsWith('ddos-guard'));
  if (cfDetected) {
    if (response.status() == 403 || response.status() == 503) {
      cfDetected = true; // Defected CloudFlare and DDoS-GUARD
    } else if (response.headers().vary && response.headers().vary.trim() == 'Accept-Encoding,User-Agent' &&
        response.headers()['content-encoding'] && response.headers()['content-encoding'].trim() == 'br') {
      cfDetected = true; // Detected Custom CloudFlare for EbookParadijs, Film-Paleis, MuziekFabriek and Puur-Hollands
    } else {
      cfDetected = false;
    }
  }

  if (cfDetected) {
    log.info('Cloudflare detected');
  } else {
    log.info('Cloudflare not detected');
    return response;
  }

  if (await findAnySelector(page, BAN_SELECTORS)) {
    throw new Error('Cloudflare has blocked this request. Probably your IP is banned for this site, check in your web browser.');
  }

  // find Cloudflare selectors
  let selectorFound = false;
  let selector: string = await findAnySelector(page, CHALLENGE_SELECTORS)
  if (selector) {
    selectorFound = true;
    log.debug(`Javascript challenge element '${selector}' detected.`)
    log.debug('Waiting for Cloudflare challenge...')

    while (true) {
      try {

        selector = await findAnySelector(page, CHALLENGE_SELECTORS)
        if (!selector) {
          // solved!
          log.debug('Challenge element not found')
          break
        } else {
          log.debug(`Javascript challenge element '${selector}' detected.`)

          // new Cloudflare Challenge #cf-please-wait
          const displayStyle = await page.evaluate((selector) => {
            return getComputedStyle(document.querySelector(selector)).getPropertyValue("display");
          }, selector);
          if (displayStyle == "none") {
            // spinner is hidden, could be a captcha or not
            log.debug('Challenge element is hidden')
            // wait until redirecting disappears
            while (true) {
              try {
                await page.waitForTimeout(1000)
                const displayStyle2 = await page.evaluate(() => {
                  return getComputedStyle(document.querySelector('#cf-spinner-redirecting')).getPropertyValue("display");
                });
                if (displayStyle2 == "none") {
                  break // hCaptcha detected
                }
              } catch (error) {
                break // redirection completed
              }
            }
            break
          } else {
            log.debug('Challenge element is visible')
          }
        }
        log.debug('Found challenge element again')

      } catch (error)
      {
        log.debug("Unexpected error: " + error);
        if (!error.toString().includes("Execution context was destroyed")) {
          break
        }
      }

      log.debug('Waiting for Cloudflare challenge...')
      await page.waitForTimeout(1000)
    }

    log.debug('Validating HTML code...')
  } else {
    log.debug(`No challenge element detected.`)
  }

  // check for CAPTCHA challenge
  if (await findAnySelector(page, CAPTCHA_SELECTORS)) {
    log.info('CAPTCHA challenge detected');
    throw new Error('FlareSolverr can not resolve CAPTCHA challenges. Since the captcha doesn\'t always appear, you may have better luck with the next request.');

    // const captchaSolver = getCaptchaSolver()
    // if (captchaSolver) {
    //     // to-do: get the params
    //     log.info('Waiting to receive captcha token to bypass challenge...')
    //     const token = await captchaSolver({
    //       url,
    //       sitekey,
    //       type: captchaType
    //     })
    //     log.debug(`Token received: ${token}`);
    //     // to-do: send the token
    //   }
    // } else {
    //   throw new Error('Captcha detected but no automatic solver is configured.');
    // }
  } else {
    if (!selectorFound)
    {
      throw new Error('No challenge selectors found, unable to proceed.')
    } else {
      log.info('Challenge solved');
    }
  }

  return response;
}

Source File: solver.ts From FlareSolverr with MIT License

4 votes

async function resolveChallenge(params: V1Request, session: SessionsCacheItem): Promise<ChallengeResolutionT | void> {
    try {
        let status = 'ok'
        let message = ''

        const page: Page = await session.browser.newPage()

        // the Puppeter timeout should be half the maxTimeout because we reload the page and wait for challenge
        // the user can set a really high maxTimeout if he wants to
        await page.setDefaultNavigationTimeout(params.maxTimeout / 2)

        // the user-agent is changed just for linux arm build
        await page.setUserAgent(sessions.getUserAgent())

        // set the proxy
        if (params.proxy) {
            log.debug(`Using proxy: ${params.proxy.url}`);
            // todo: credentials are not working
            // if (params.proxy.username) {
            //     await page.authenticate({
            //         username: params.proxy.username,
            //         password: params.proxy.password
            //     });
            // }
        }

        // go to the page
        log.debug(`Navigating to... ${params.url}`)
        let response: HTTPResponse = await gotoPage(params, page);

        // set cookies
        if (params.cookies) {
            for (const cookie of params.cookies) {
                // the other fields in the cookie can cause issues
                await page.setCookie({
                    "name": cookie.name,
                    "value": cookie.value
                })
            }
            // reload the page
            response = await gotoPage(params, page);
        }

        // log html in debug mode
        log.html(await page.content())

        // detect protection services and solve challenges
        try {
            response = await cloudflareProvider(params.url, page, response);

            // is response is ok
            // reload the page to be sure we get the real page
            log.debug("Reloading the page")
            try {
                response = await gotoPage(params, page);
            } catch (e) {
                log.warn("Page not reloaded (do not report!): Cause: " + e.toString())
            }

        } catch (e) {
            status = "error";
            message = "Cloudflare " + e.toString();
        }

        const payload: ChallengeResolutionT = {
            status,
            message,
            result: {
                url: page.url(),
                status: response.status(),
                headers: response.headers(),
                response: null,
                cookies: await page.cookies(),
                userAgent: sessions.getUserAgent()
            }
        }

        if (params.returnOnlyCookies) {
            payload.result.headers = null;
            payload.result.userAgent = null;
        } else {
            payload.result.response = await page.content()
        }

        // make sure the page is closed because if it isn't and error will be thrown
        // when a user uses a temporary session, the browser make be quit before
        // the page is properly closed.
        await page.close()

        return payload
    } catch (e) {
        log.error("Unexpected error: " + e);
        throw e;
    }
}

Source File: browser.test.ts From web with MIT License

4 votes

describe('browser tests', function () {
  this.timeout(10000);

  let browser: Browser;

  before(async () => {
    browser = await launchPuppeteer();
  });

  after(async () => {
    await browser.close();
  });

  it('should bubble when bubbles is true', async () => {
    const { server, host } = await createTestServer({
      rootDir: __dirname,
      plugins: [
        mockFiles({
          '/foo.html': '<script src="/foo.js" type="module"></script>',
          '/foo.js': `import '/bar.js'; import.meta.hot.accept();`,
          '/bar.js': `import.meta.hot.accept({ bubbles: true })`,
        }),
        hmrPlugin(),
      ],
    });
    const { fileWatcher, webSockets } = server;
    const stub = stubMethod(webSockets, 'send');
    const page = await browser.newPage();
    try {
      await page.goto(`${host}/foo.html`, { waitUntil: 'networkidle0' });
      fileWatcher.emit('change', pathUtil.join(__dirname, '/bar.js'));

      expect(stub.callCount).to.equal(2);
      expect(stub.getCall(0)!.args[0]).to.equal(
        JSON.stringify({
          type: 'hmr:update',
          url: '/bar.js',
        }),
      );
      expect(stub.getCall(1)!.args[0]).to.equal(
        JSON.stringify({
          type: 'hmr:update',
          url: '/foo.js',
        }),
      );
    } finally {
      await page.close();
      await server.stop();
    }
  });

  it('should hot replace a module', async () => {
    const files = {
      '/foo.html': '<script src="/foo.js" type="module"></script>',
      '/foo.js':
        'import.meta.hot.accept(); document.body.appendChild(document.createTextNode(" a "));',
    };
    const { server, host } = await createTestServer({
      rootDir: __dirname,
      plugins: [mockFiles(files), hmrPlugin()],
    });
    const page = await browser.newPage();
    const errors = trackErrors(page);

    try {
      await page.goto(`${host}/foo.html`, { waitUntil: 'networkidle0' });
      expectIncludes(await page.content(), '<body> a </body>');

      files['/foo.js'] = files['/foo.js'].replace('" a "', '" b "');
      server.fileWatcher.emit('change', pathUtil.join(__dirname, '/foo.js'));
      await page.waitForResponse((r: HTTPResponse) => r.url().startsWith(`${host}/foo.js`));
      expectIncludes(await page.content(), '<body> a  b </body>');

      for (const error of errors) {
        throw error;
      }
    } finally {
      await page.close();
      await server.stop();
    }
  });

  it('should hot replace a bubbled module', async () => {
    const files = {
      '/foo.html': '<script src="/foo.js" type="module"></script>',
      '/foo.js':
        'import bar from "/bar.js"; import.meta.hot.accept(); document.body.appendChild(document.createTextNode(bar));',
      '/bar.js': 'export default " a ";',
    };
    const { server, host } = await createTestServer({
      rootDir: __dirname,
      plugins: [mockFiles(files), hmrPlugin()],
    });
    const page = await browser.newPage();
    const errors = trackErrors(page);

    try {
      await page.goto(`${host}/foo.html`, { waitUntil: 'networkidle0' });
      expectIncludes(await page.content(), '<body> a </body>');

      files['/bar.js'] = 'export default " b ";';
      server.fileWatcher.emit('change', pathUtil.join(__dirname, '/bar.js'));
      await page.waitForResponse((r: HTTPResponse) => r.url().startsWith(`${host}/bar.js`));
      await new Promise(r => setTimeout(r, 1000));
      expectIncludes(await page.content(), '<body> a  b </body>');

      for (const error of errors) {
        throw error;
      }
    } finally {
      await page.close();
      await server.stop();
    }
  });

  it('hot replaces multiple bubbled modules', async () => {
    const files = {
      '/foo.html': '<script type="module">import "/foo.js"; import "/bar.js";</script>',
      '/foo.js':
        'import baz from "/baz.js"; import.meta.hot.accept(); document.body.appendChild(document.createTextNode(" foo " + baz));',
      '/bar.js':
        'import baz from "/baz.js"; import.meta.hot.accept(); document.body.appendChild(document.createTextNode(" bar " + baz));',
      '/baz.js': 'export default " a ";',
    };
    const { server, host } = await createTestServer({
      rootDir: __dirname,
      plugins: [mockFiles(files), hmrPlugin()],
    });
    const page = await browser.newPage();
    const errors = trackErrors(page);

    try {
      await page.goto(`${host}/foo.html`, { waitUntil: 'networkidle0' });
      // expectIncludes(await page.content(), '<body> foo  a  bar  a </body>');

      files['/baz.js'] = 'export default " b ";';
      server.fileWatcher.emit('change', pathUtil.join(__dirname, '/baz.js'));
      await Promise.all([
        page.waitForResponse((r: HTTPResponse) => r.url().startsWith(`${host}/foo.js`)),
        page.waitForResponse((r: HTTPResponse) => r.url().startsWith(`${host}/bar.js`)),
        page.waitForResponse((r: HTTPResponse) => r.url().startsWith(`${host}/baz.js`)),
      ]);
      await new Promise(r => setTimeout(r, 1000));
      expectIncludes(await page.content(), '<body> foo  a  bar  a  foo  b  bar  b </body>');

      for (const error of errors) {
        throw error;
      }
    } finally {
      await page.close();
      await server.stop();
    }
  });

  it('reloads the page when a module has no hot replacable parent', async () => {
    const files = {
      '/foo.html':
        '<script src="/foo.js" type="module"></script><script src="/baz.js" type="module"></script>',
      '/foo.js':
        'import bar from "/bar.js"; import.meta.hot.accept(); document.body.appendChild(document.createTextNode(bar));',
      '/bar.js': 'export default " a ";',
      '/baz.js': 'document.body.appendChild(document.createTextNode(" b "));',
    };
    const { server, host } = await createTestServer({
      rootDir: __dirname,
      plugins: [mockFiles(files), hmrPlugin()],
    });
    const page = await browser.newPage();
    const errors = trackErrors(page);

    try {
      await page.goto(`${host}/foo.html`, { waitUntil: 'networkidle0' });
      await page.evaluate('document.body.appendChild(document.createTextNode(" c "))');
      expectIncludes(await page.content(), '<body> a  b  c </body>');

      server.fileWatcher.emit('change', pathUtil.join(__dirname, '/baz.js'));
      await page.waitForNavigation({ waitUntil: 'networkidle0' });
      expectIncludes(await page.content(), '<body> a  b </body>');

      for (const error of errors) {
        throw error;
      }
    } finally {
      await page.close();
      await server.stop();
    }
  });
});

puppeteer#HTTPResponse TypeScript Examples