/*--------------------------------------------------------------------------------------------- * Copyright (c) Microsoft Corporation. All rights reserved. * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ //@ts-check 'use strict'; const path = require('path'); const glob = require('glob'); const events = require('events'); const mocha = require('mocha'); const createStatsCollector = require('mocha/lib/stats-collector'); const MochaJUnitReporter = require('mocha-junit-reporter'); const url = require('url'); const minimatch = require('minimatch'); const fs = require('fs'); const playwright = require('@playwright/test'); const { applyReporter } = require('../reporter'); const yaserver = require('yaserver'); const http = require('http'); const { randomBytes } = require('crypto'); const minimist = require('minimist'); const { promisify } = require('node:util'); /** * @type {{ * run: string; * grep: string; * runGlob: string; * browser: string; * reporter: string; * 'reporter-options': string; * tfs: string; * build: boolean; * debug: boolean; * sequential: boolean; * help: boolean; * }} */ const args = minimist(process.argv.slice(2), { boolean: ['build', 'debug', 'sequential', 'help'], string: ['run', 'grep', 'runGlob', 'browser', 'reporter', 'reporter-options', 'tfs'], default: { build: false, browser: ['chromium', 'firefox', 'webkit'], reporter: process.platform === 'win32' ? 'list' : 'spec', 'reporter-options': '' }, alias: { grep: ['g', 'f'], runGlob: ['glob', 'runGrep'], debug: ['debug-browser'], help: 'h' }, describe: { build: 'run with build output (out-build)', run: 'only run tests matching ', grep: 'only run tests matching ', debug: 'do not run browsers headless', sequential: 'only run suites for a single browser at a time', browser: 'browsers in which tests should run', reporter: 'the mocha reporter', 'reporter-options': 'the mocha reporter options', tfs: 'tfs', help: 'show the help' } }); if (args.help) { console.log(`Usage: node ${process.argv[1]} [options] Options: --build run with build output (out-build) --run only run tests matching --grep, -g, -f only run tests matching --debug, --debug-browser do not run browsers headless --sequential only run suites for a single browser at a time --browser browsers in which tests should run --reporter the mocha reporter --reporter-options the mocha reporter options --tfs tfs --help, -h show the help`); process.exit(0); } const isDebug = !!args.debug; const withReporter = (function () { if (args.tfs) { { return (browserType, runner) => { new mocha.reporters.Spec(runner); new MochaJUnitReporter(runner, { reporterOptions: { testsuitesTitle: `${args.tfs} ${process.platform}`, mochaFile: process.env.BUILD_ARTIFACTSTAGINGDIRECTORY ? path.join(process.env.BUILD_ARTIFACTSTAGINGDIRECTORY, `test-results/${process.platform}-${process.arch}-${browserType}-${args.tfs.toLowerCase().replace(/[^\w]/g, '-')}-results.xml`) : undefined } }); }; } } else { return (_, runner) => applyReporter(runner, args); } })(); const outdir = args.build ? 'out-build' : 'out'; const rootDir = path.resolve(__dirname, '..', '..', '..'); const out = path.join(rootDir, `${outdir}`); function ensureIsArray(a) { return Array.isArray(a) ? a : [a]; } const testModules = (async function () { const excludeGlob = '**/{node,electron-sandbox,electron-main,electron-utility}/**/*.test.js'; let isDefaultModules = true; let promise; if (args.run) { // use file list (--run) isDefaultModules = false; promise = Promise.resolve(ensureIsArray(args.run).map(file => { file = file.replace(/^src/, 'out'); file = file.replace(/\.ts$/, '.js'); return path.relative(out, file); })); } else { // glob patterns (--glob) const defaultGlob = '**/*.test.js'; const pattern = args.runGlob || defaultGlob; isDefaultModules = pattern === defaultGlob; promise = new Promise((resolve, reject) => { glob(pattern, { cwd: out }, (err, files) => { if (err) { reject(err); } else { resolve(files); } }); }); } return promise.then(files => { const modules = []; for (const file of files) { if (!minimatch(file, excludeGlob)) { modules.push(file.replace(/\.js$/, '')); } else if (!isDefaultModules) { console.warn(`DROPPONG ${file} because it cannot be run inside a browser`); } } return modules; }); })(); function consoleLogFn(msg) { const type = msg.type(); const candidate = console[type]; if (candidate) { return candidate; } if (type === 'warning') { return console.warn; } return console.log; } async function createServer() { // Demand a prefix to avoid issues with other services on the // machine being able to access the test server. const prefix = '/' + randomBytes(16).toString('hex'); const serveStatic = await yaserver.createServer({ rootDir }); /** Handles a request for a remote method call, invoking `fn` and returning the result */ const remoteMethod = async (req, response, fn) => { const params = await new Promise((resolve, reject) => { const body = []; req.on('data', chunk => body.push(chunk)); req.on('end', () => resolve(JSON.parse(Buffer.concat(body).toString()))); req.on('error', reject); }); try { const result = await fn(...params); response.writeHead(200, { 'Content-Type': 'application/json' }); response.end(JSON.stringify(result)); } catch (err) { response.writeHead(500); response.end(err.message); } }; const server = http.createServer((request, response) => { if (!request.url?.startsWith(prefix)) { return response.writeHead(404).end(); } // rewrite the URL so the static server can handle the request correctly request.url = request.url.slice(prefix.length); function massagePath(p) { // TODO@jrieken FISHY but it enables snapshot // in ESM browser tests p = String(p).replace(/\\/g, '/').replace(prefix, rootDir); return p; } switch (request.url) { case '/remoteMethod/__readFileInTests': return remoteMethod(request, response, p => fs.promises.readFile(massagePath(p), 'utf-8')); case '/remoteMethod/__writeFileInTests': return remoteMethod(request, response, (p, contents) => fs.promises.writeFile(massagePath(p), contents)); case '/remoteMethod/__readDirInTests': return remoteMethod(request, response, p => fs.promises.readdir(massagePath(p))); case '/remoteMethod/__unlinkInTests': return remoteMethod(request, response, p => fs.promises.unlink(massagePath(p))); case '/remoteMethod/__mkdirPInTests': return remoteMethod(request, response, p => fs.promises.mkdir(massagePath(p), { recursive: true })); default: return serveStatic.handle(request, response); } }); return new Promise((resolve, reject) => { server.listen(0, 'localhost', () => { resolve({ dispose: () => server.close(), // @ts-ignore url: `http://localhost:${server.address().port}${prefix}` }); }); server.on('error', reject); }); } async function runTestsInBrowser(testModules, browserType) { const server = await createServer(); const browser = await playwright[browserType].launch({ headless: !Boolean(args.debug), devtools: Boolean(args.debug) }); const context = await browser.newContext(); const page = await context.newPage(); const target = new URL(server.url + '/test/unit/browser/renderer.html'); target.searchParams.set('baseUrl', url.pathToFileURL(path.join(rootDir, 'src')).toString()); if (args.build) { target.searchParams.set('build', 'true'); } if (process.env.BUILD_ARTIFACTSTAGINGDIRECTORY) { target.searchParams.set('ci', 'true'); } // append CSS modules as query-param await promisify(require('glob'))('**/*.css', { cwd: out }).then(async cssModules => { const cssData = await new Response((await new Response(cssModules.join(',')).blob()).stream().pipeThrough(new CompressionStream('gzip'))).arrayBuffer(); target.searchParams.set('_devCssData', Buffer.from(cssData).toString('base64')); }); const emitter = new events.EventEmitter(); await page.exposeFunction('mocha_report', (type, data1, data2) => { emitter.emit(type, data1, data2); }); await page.goto(target.href); if (args.build) { const nlsMessages = await fs.promises.readFile(path.join(out, 'nls.messages.json'), 'utf8'); await page.evaluate(value => { // when running from `out-build`, ensure to load the default // messages file, because all `nls.localize` calls have their // english values removed and replaced by an index. // @ts-ignore globalThis._VSCODE_NLS_MESSAGES = JSON.parse(value); }, nlsMessages); } page.on('console', async msg => { consoleLogFn(msg)(msg.text(), await Promise.all(msg.args().map(async arg => await arg.jsonValue()))); }); withReporter(browserType, new EchoRunner(emitter, browserType.toUpperCase())); // collection failures for console printing const failingModuleIds = []; const failingTests = []; emitter.on('fail', (test, err) => { failingTests.push({ title: test.fullTitle, message: err.message }); if (err.stack) { const regex = /(vs\/.*\.test)\.js/; for (const line of String(err.stack).split('\n')) { const match = regex.exec(line); if (match) { failingModuleIds.push(match[1]); return; } } } }); try { // @ts-expect-error await page.evaluate(opts => loadAndRun(opts), { modules: testModules, grep: args.grep, }); } catch (err) { console.error(err); } if (!isDebug) { server?.dispose(); await browser.close(); } if (failingTests.length > 0) { let res = `The followings tests are failing:\n - ${failingTests.map(({ title, message }) => `${title} (reason: ${message})`).join('\n - ')}`; if (failingModuleIds.length > 0) { res += `\n\nTo DEBUG, open ${browserType.toUpperCase()} and navigate to ${target.href}?${failingModuleIds.map(module => `m=${module}`).join('&')}`; } return `${res}\n`; } } class EchoRunner extends events.EventEmitter { constructor(event, title = '') { super(); createStatsCollector(this); event.on('start', () => this.emit('start')); event.on('end', () => this.emit('end')); event.on('suite', (suite) => this.emit('suite', EchoRunner.deserializeSuite(suite, title))); event.on('suite end', (suite) => this.emit('suite end', EchoRunner.deserializeSuite(suite, title))); event.on('test', (test) => this.emit('test', EchoRunner.deserializeRunnable(test))); event.on('test end', (test) => this.emit('test end', EchoRunner.deserializeRunnable(test))); event.on('hook', (hook) => this.emit('hook', EchoRunner.deserializeRunnable(hook))); event.on('hook end', (hook) => this.emit('hook end', EchoRunner.deserializeRunnable(hook))); event.on('pass', (test) => this.emit('pass', EchoRunner.deserializeRunnable(test))); event.on('fail', (test, err) => this.emit('fail', EchoRunner.deserializeRunnable(test, title), EchoRunner.deserializeError(err))); event.on('pending', (test) => this.emit('pending', EchoRunner.deserializeRunnable(test))); } static deserializeSuite(suite, titleExtra) { return { root: suite.root, suites: suite.suites, tests: suite.tests, title: titleExtra && suite.title ? `${suite.title} - /${titleExtra}/` : suite.title, titlePath: () => suite.titlePath, fullTitle: () => suite.fullTitle, timeout: () => suite.timeout, retries: () => suite.retries, slow: () => suite.slow, bail: () => suite.bail }; } static deserializeRunnable(runnable, titleExtra) { return { title: runnable.title, fullTitle: () => titleExtra && runnable.fullTitle ? `${runnable.fullTitle} - /${titleExtra}/` : runnable.fullTitle, titlePath: () => runnable.titlePath, async: runnable.async, slow: () => runnable.slow, speed: runnable.speed, duration: runnable.duration, currentRetry: () => runnable.currentRetry, }; } static deserializeError(err) { const inspect = err.inspect; err.inspect = () => inspect; return err; } } testModules.then(async modules => { // run tests in selected browsers const browserTypes = Array.isArray(args.browser) ? args.browser : [args.browser]; let messages = []; let didFail = false; try { if (args.sequential) { for (const browserType of browserTypes) { messages.push(await runTestsInBrowser(modules, browserType)); } } else { messages = await Promise.all(browserTypes.map(async browserType => { return await runTestsInBrowser(modules, browserType); })); } } catch (err) { console.error(err); if (!isDebug) { process.exit(1); } } // aftermath for (const msg of messages) { if (msg) { didFail = true; console.log(msg); } } if (!isDebug) { process.exit(didFail ? 1 : 0); } }).catch(err => { console.error(err); });