Playwright command

lib/scenario_runner.py

@@ -1589,77 +1589,95 @@ def _run_flows(self):
                     if 'note' in cmd_obj:
                         self.__notes_helper.add_note( note=cmd_obj['note'], detail_name=flow['container'], timestamp=int(time.time_ns() / 1_000))
 
-                    if cmd_obj['type'] == 'console':
-                        print(TerminalColors.HEADER, '\nConsole command', cmd_obj['command'], 'on container', flow['container'], TerminalColors.ENDC)
+                    print(TerminalColors.HEADER, '\n', cmd_obj['type'], 'command:', cmd_obj['command'], 'on container', flow['container'], TerminalColors.ENDC)
 
-                        docker_exec_command = ['docker', 'exec']
+                    docker_exec_command = ['docker', 'exec']
+                    docker_exec_command.append(flow['container'])
+                    stderr_behaviour = stdout_behaviour = subprocess.DEVNULL
+                    if cmd_obj.get('log-stdout', True):
+                        stdout_behaviour = subprocess.PIPE
+                    if cmd_obj.get('log-stderr', True):
+                        stderr_behaviour = subprocess.PIPE
+
+                    if cmd_obj['type'] == 'playwright':
+                        docker_exec_command.append(cmd_obj.get('shell', 'sh'))
+                        docker_exec_command.append('-c')
+                        escaped_command = cmd_obj['command'].replace("'", "\\'")
+                        docker_exec_command.append(f"echo '{escaped_command}' > /tmp/playwright-ipc-commands")
+
+                    elif cmd_obj['type'] == 'console':
 
-                        docker_exec_command.append(flow['container'])
                         if shell := cmd_obj.get('shell', False):
                             docker_exec_command.append(shell)
                             docker_exec_command.append('-c')
                             docker_exec_command.append(cmd_obj['command'])
                         else:
                             docker_exec_command.extend(shlex.split(cmd_obj['command']))
+                    else:
+                        raise RuntimeError('Unknown command type in flow: ', cmd_obj['type'])
 
+
+                    if cmd_obj.get('detach', False) is True and cmd_obj['type'] == 'playwright':
+                        raise ValueError('Playwright commands cannot be executed detached, as they must be awaited')
+                    elif cmd_obj.get('detach', False) is True:
+                        print('Executing process asynchronously and detaching ...')
                         # Note: In case of a detach wish in the usage_scenario.yml:
                         # We are NOT using the -d flag from docker exec, as this prohibits getting the stdout.
-                        # Since Popen always make the process asynchronous we can leverage this to emulate a detached
-                        # behavior
-
-                        stderr_behaviour = stdout_behaviour = subprocess.DEVNULL
-                        if cmd_obj.get('log-stdout', True):
-                            stdout_behaviour = subprocess.PIPE
-                        if cmd_obj.get('log-stderr', True):
-                            stderr_behaviour = subprocess.PIPE
-
-
-                        if cmd_obj.get('detach', False) is True:
-                            print('Executing process asynchronously and detaching ...')
-                            #pylint: disable=consider-using-with,subprocess-popen-preexec-fn
-                            ps = subprocess.Popen(
-                                docker_exec_command,
-                                stderr=stderr_behaviour,
-                                stdout=stdout_behaviour,
-                                preexec_fn=os.setsid,
-                                encoding='UTF-8',
-                                errors='replace',
-                            )
-                            if stderr_behaviour == subprocess.PIPE:
-                                os.set_blocking(ps.stderr.fileno(), False)
-                            if  stdout_behaviour == subprocess.PIPE:
-                                os.set_blocking(ps.stdout.fileno(), False)
-
-                            ps_to_kill_tmp.append({'ps': ps, 'cmd': cmd_obj['command'], 'ps_group': False})
-                        else:
-                            print('Executing process synchronously.')
-                            if self._measurement_flow_process_duration:
-                                print(f"Alloting {self._measurement_flow_process_duration}s runtime ...")
-
-                            ps = subprocess.run(
-                                docker_exec_command,
-                                stderr=stderr_behaviour,
-                                stdout=stdout_behaviour,
-                                encoding='UTF-8',
-                                errors='replace',
-                                check=False, # cause it will be checked later and also ignore-errors checked
-                                timeout=self._measurement_flow_process_duration,
-                            )
-
-                        ps_to_read_tmp.append({
-                            'cmd': docker_exec_command,
-                            'ps': ps,
-                            'container_name': flow['container'],
-                            'read-notes-stdout': cmd_obj.get('read-notes-stdout', False),
-                            'ignore-errors': cmd_obj.get('ignore-errors', False),
-                            'read-sci-stdout': cmd_obj.get('read-sci-stdout', False),
-                            'detail_name': flow['container'],
-                            'detach': cmd_obj.get('detach', False),
-                        })
-
+                        # Since Popen always make the process asynchronous we can leverage this to emulate a detached behavior
+
+                        #pylint: disable=consider-using-with,subprocess-popen-preexec-fn
+                        ps = subprocess.Popen(
+                            docker_exec_command,
+                            stderr=stderr_behaviour,
+                            stdout=stdout_behaviour,
+                            preexec_fn=os.setsid,
+                            encoding='UTF-8',
+                            errors='replace',
+                        )
+                        if stderr_behaviour == subprocess.PIPE:
+                            os.set_blocking(ps.stderr.fileno(), False)
+                        if  stdout_behaviour == subprocess.PIPE:
+                            os.set_blocking(ps.stdout.fileno(), False)
 
+                        ps_to_kill_tmp.append({'ps': ps, 'cmd': cmd_obj['command'], 'ps_group': False})
                     else:
-                        raise RuntimeError('Unknown command type in flow: ', cmd_obj['type'])
+                        print('Executing process synchronously.')
+                        if self._measurement_flow_process_duration:
+                            print(f"Alloting {self._measurement_flow_process_duration}s runtime ...")
+
+                        ps = subprocess.run(
+                            docker_exec_command,
+                            stderr=stderr_behaviour,
+                            stdout=stdout_behaviour,
+                            encoding='UTF-8',
+                            errors='replace',
+                            check=False, # cause it will be checked later and also ignore-errors checked
+                            timeout=self._measurement_flow_process_duration,
+                        )
+
+                    ps_to_read_tmp.append({
+                        'cmd': docker_exec_command,
+                        'ps': ps,
+                        'container_name': flow['container'],
+                        'read-notes-stdout': cmd_obj.get('read-notes-stdout', False),
+                        'ignore-errors': cmd_obj.get('ignore-errors', False),
+                        'read-sci-stdout': cmd_obj.get('read-sci-stdout', False),
+                        'detail_name': flow['container'],
+                        'detach': cmd_obj.get('detach', False),
+                    })
+
+                    # we need to check the ready IPC endpoint to find out if the process is done
+                    # this command will block until something is received
+                    if cmd_obj['type'] == 'playwright':
+                        print("Awaiting Playwright function return")
+                        ps = subprocess.run(
+                            ['docker', 'exec', flow['container'], 'cat', '/tmp/playwright-ipc-ready'],
+                            check=True,
+                            stdout=subprocess.PIPE,
+                            stderr=subprocess.PIPE,
+                            timeout=60, # 60 seconds should be reasonable for any playwright command we know
+                        )
+
 
                     if self._debugger.active:
                         self._debugger.pause('Waiting to start next command in flow')

lib/schema_checker.py

@@ -177,7 +177,7 @@ def check_usage_scenario(self, usage_scenario):
                 'name': And(str, Use(self.not_empty), Regex(r'^[\.\s0-9a-zA-Z_\(\)-]+$')),
                 'container': And(str, Use(self.not_empty), Use(self.contains_no_invalid_chars)),
                 'commands': [{
-                    'type': 'console',
+                    'type': Or('console', 'playwright'),
                     'command': And(str, Use(self.not_empty)),
                     Optional('detach'): bool,
                     Optional('note'): And(str, Use(self.not_empty)),

templates/website/compose.yml

@@ -1,13 +1,19 @@
 services:
-  gcb-playwright:
-    image: greencoding/gcb_playwright:v20
+  playwright-nodejs:
+    image: mcr.microsoft.com/playwright:v1.55.0-noble
 #    volumes:
 #       - /tmp/.X11-unix:/tmp/.X11-unix # for debugging in non-headless mode
     environment:
       DISPLAY: ":0" # for debugging in non-headless mode
     depends_on:
       squid:
         condition: service_healthy
+    setup-commands:
+        # install playwright libraries
+      - command: mkdir /tmp/energy-tests
+      - command: cp -R /tmp/repo/. /tmp/energy-tests
+      - command: cd /tmp/energy-tests && npm init -y && npm install playwright
+        shell: bash
 
   squid:
     image: greencoding/squid_reverse_proxy:v4

templates/website/playwright-ipc.js

@@ -0,0 +1,89 @@
+import { firefox, chromium } from "playwright";
+import fs from "fs";
+import { execSync } from "child_process";
+
+
+const defaultProxy = { server: "http://squid:3128" };
+const contextOptions = {
+  viewport: { width: 1280, height: 800 },
+  ignoreHTTPSErrors: true, // <--- disables SSL check as we funnel requests through proxy
+  timeout: 5000,
+};
+
+function logNote(message) {
+  const timestamp = String(BigInt(Date.now()) * 1000000n).slice(0, 16);
+  console.log(`${timestamp} ${message}`);
+}
+
+async function startFifoReader(fifoPath, callback) {
+  function openStream() {
+    const stream = fs.createReadStream(fifoPath, { encoding: "utf-8" });
+    stream.on("data", (chunk) => callback(chunk.trim()));
+    stream.on("end", () => {
+      // Writer closed FIFO, reopen it
+      openStream();
+    });
+    stream.on("error", (err) => {
+      console.error(err);
+      throw err
+      // setTimeout(openStream, 100); // reopening is not really an option for us as we run inside container
+    });
+  }
+  openStream();
+}
+
+
+// Test 1 - BRANCH: Launch Branch website with no cookies
+async function run(browserName, headless, proxy) {
+  let browser = null;
+  const launchOptions = { headless, proxy };
+
+  if (browserName === "firefox") {
+    browser = await firefox.launch(launchOptions);
+  } else {
+    browser = await chromium.launch({
+      ...launchOptions,
+      args: headless ? ["--headless=new"] : [],
+    });
+  }
+
+  let context = await browser.newContext(contextOptions);
+  await context.clearCookies();
+  let page = await context.newPage()
+
+  execSync(`mkfifo /tmp/playwright-ipc-ready`); // signal that browser is launched
+  execSync(`mkfifo /tmp/playwright-ipc-commands`); // create pipe to get commands
+
+  await startFifoReader("/tmp/playwright-ipc-commands", async (data) => {
+      if (data == 'end') {
+          await browser.close()
+          fs.writeFileSync("/tmp/playwright-ipc-ready", "ready", "utf-8");   // signal that browser is ready although
+          process.exit(0)
+      } else {
+          console.log('Evaluating', data);
+          await eval(`(async () => { ${data} })()`);
+          fs.writeFileSync("/tmp/playwright-ipc-ready", "ready", "utf-8");   // signal that browser is ready for next command
+      }
+  });
+
+};
+
+// CLI args
+const argv = process.argv;
+const args = {};
+for (let i = 2; i < argv.length; i++) {
+  if (argv[i] === "--browser" && argv[i + 1]) {
+    args.browser = argv[++i];
+  } else if (argv[i] === "--headless" && argv[i + 1]) {
+    args.headless = argv[++i].toLowerCase() === "true";
+  } else if (argv[i] === "--proxy" && argv[i + 1]) {
+    args.proxy = argv[++i] === "null" ? null : { server: argv[i] };
+  }
+}
+
+await run(
+  (args.browser || "chromium").toLowerCase(),
+  args.headless !== undefined ? args.headless : true,
+  args.proxy !== undefined ? args.proxy : defaultProxy
+);
+

templates/website/usage_scenario.yml

@@ -8,51 +8,58 @@ sci:
 
 compose-file: !include compose.yml
 
-services:
-  gcb-playwright:
-    setup-commands:
-      - command: mkfifo /tmp/my_fifo
-      - command: python3 /tmp/repo/templates/website/visit.py --browser chromium
+flow:
+  - name: Starting browser IPC
+    container: playwright-nodejs
+    commands:
+      - type: console
+        command: node /tmp/energy-tests/templates/website/playwright-ipc.js --headless true --browser firefox
+        note: Starting browser in background process with IPC
         detach: true
-      - command: until [ -f "/tmp/browser_ready" ]; do sleep 1; done && echo "Browser ready!"
-        shell: bash
-      - command: echo '__GMT_VAR_PAGE__' > /tmp/my_fifo # warmup
+      - type: console
+        command: until [ -p "/tmp/playwright-ipc-ready" ]; do sleep 1; done && echo "Browser ready!"
         shell: bash
-      - command: sleep '__GMT_VAR_SLEEP__' # we need same sleep as later as some resources might only be fetched from the website after staying on the page for a while
+        note: Waiting for website stepper loop to start by monitoring rendevous file endpoint
+
+  - name: Warmup and Caching
+    container: playwright-nodejs
+    commands:
+      - type: playwright
+        command: await page.goto("__GMT_VAR_PAGE__");
+      - type: console
+        command: sleep '__GMT_VAR_SLEEP__'
+      - type: playwright
+        command: await context.close()
+      - type: playwright
+        command: context = await browser.newContext(contextOptions);
+      - type: playwright
+        command: page = await context.newPage()
 
-flow:
   - name: Dump Log (Warmup)
     container: squid
     commands:
       - type: console
         command: cat /apps/squid/var/logs/access.log
-        log-stdout: true
-        log-stderr: true
       - type: console
         command: grep 'TCP_MISS/' /apps/squid/var/logs/access.log #validate that TCP_MISSes present
       - type: console
         command: echo > /apps/squid/var/logs/access.log
         shell: bash
-        log-stdout: true
-        log-stderr: true
+
   - name: Load and idle
-    container: gcb-playwright
+    container: playwright-nodejs
     commands:
+      - type: playwright
+        command: await page.goto("__GMT_VAR_PAGE__");
       - type: console
-        shell: bash
-        command: "echo '__GMT_VAR_PAGE__' > /tmp/my_fifo && sleep __GMT_VAR_SLEEP__"
-        read-notes-stdout: true
-        log-stdout: true
-        log-stderr: true
+        command: sleep '__GMT_VAR_SLEEP__'
 
   - name: Dump Log (Load and idle)
     container: squid
     commands:
       - type: console
         command: cat /apps/squid/var/logs/access.log
         read-notes-stdout: true
-        log-stdout: true
-        log-stderr: true
       - type: console
         command: grep 'TCP_MEM_HIT/' /apps/squid/var/logs/access.log #validate that TCP_MEM_HITs present
         # This is sadly too strict. Pages fingerprint you all the time and a new request might get a new fingerprint that will not be cached