Remove html escape

api/api_helpers.py

@@ -4,7 +4,6 @@
 
 from collections import OrderedDict
 from functools import cache
-from html import escape as html_escape
 import typing
 import uuid
 
@@ -15,7 +14,6 @@
 from fastapi.exceptions import RequestValidationError
 import numpy as np
 import scipy.stats
-from pydantic import BaseModel
 
 from psycopg import sql
 
@@ -106,46 +104,6 @@ def is_valid_uuid(val):
     except ValueError:
         return False
 
-def html_escape_multi(item):
-    """Replace special characters "'", "\"", "&", "<" and ">" to HTML-safe sequences."""
-    if item is None:
-        return None
-
-    if isinstance(item, str):
-        return html_escape(item)
-
-    if isinstance(item, list):
-        return [html_escape_multi(element) for element in item]
-
-    if isinstance(item, dict):
-        for key, value in item.items():
-            if isinstance(value, str):
-                item[key] = html_escape(value)
-            elif isinstance(value, dict):
-                item[key] = html_escape_multi(value)
-            elif isinstance(value, list):
-                item[key] = [
-                    html_escape_multi(item)
-                    if isinstance(item, dict)
-                    else html_escape(item)
-                    if isinstance(item, str)
-                    else item
-                    for item in value
-                ]
-        return item
-
-    if isinstance(item, BaseModel):
-        item_copy = item.model_copy(deep=True)
-        # we ignore keys that begin with model_ because pydantic v2 renamed a lot of their fields from __fields to model_fields:
-        # https://docs.pydantic.dev/dev-v2/migration/#changes-to-pydanticbasemodel
-        # This could cause an error if we ever make a BaseModel that has keys that begin with model_
-        keys = [key for key in dir(item_copy) if not key.startswith('_') and not key.startswith('model_') and not callable(getattr(item_copy, key))]
-        for key in keys:
-            setattr(item_copy, key, html_escape_multi(getattr(item_copy, key)))
-        return item_copy
-
-    return item
-
 def get_machine_list():
     query = """
         WITH timings as (

api/eco_ci.py

@@ -5,7 +5,7 @@
 from fastapi.responses import ORJSONResponse
 from fastapi.exceptions import RequestValidationError
 
-from api.api_helpers import authenticate, html_escape_multi, get_connecting_ip, convert_value
+from api.api_helpers import authenticate, get_connecting_ip, convert_value
 from api.object_specifications import CI_Measurement
 
 import anybadge
@@ -30,8 +30,6 @@ async def post_ci_measurement_add(
     user: User = Depends(authenticate) # pylint: disable=unused-argument
     ):
 
-    measurement = html_escape_multi(measurement)
-
     params = [measurement.energy_uj, measurement.repo, measurement.branch,
             measurement.workflow, measurement.run_id, measurement.label, measurement.source, measurement.cpu,
             measurement.commit_hash, measurement.duration_us, measurement.cpu_util_avg, measurement.workflow_name,

api/main.py

@@ -14,7 +14,7 @@
 from starlette.exceptions import HTTPException as StarletteHTTPException
 from starlette.datastructures import Headers as StarletteHeaders
 
-from api.api_helpers import authenticate, html_escape_multi
+from api.api_helpers import authenticate
 
 from lib.global_config import GlobalConfig
 from lib import error_helpers
@@ -144,7 +144,6 @@ async def get_user_settings(user: User = Depends(authenticate)):
 
 @app.put('/v1/user/setting')
 async def update_user_setting(setting: UserSetting, user: User = Depends(authenticate)):
-    setting = html_escape_multi(setting)
 
     try:
         user.change_setting(setting.name, setting.value)

api/scenario_runner.py

@@ -14,7 +14,7 @@
 from api.object_specifications import Software, JobChange
 from api.api_helpers import (ORJSONResponseObjKeep, add_phase_stats_statistics,
                          determine_comparison_case,get_comparison_details,
-                         html_escape_multi, get_phase_stats, get_phase_stats_object, check_run_failed,
+                         get_phase_stats, get_phase_stats_object, check_run_failed,
                          is_valid_uuid, convert_value, get_timeline_query,
                          get_run_info, get_machine_list, get_artifact, store_artifact,
                          authenticate, check_int_field_api)
@@ -168,8 +168,7 @@ async def get_notes(run_id, user: User = Depends(authenticate)):
     if data is None or data == []:
         return Response(status_code=204) # No-Content
 
-    escaped_data = [html_escape_multi(note) for note in data]
-    return ORJSONResponseObjKeep({'success': True, 'data': escaped_data})
+    return ORJSONResponseObjKeep({'success': True, 'data': data})
 
 
 @router.get('/v1/warnings/{run_id}')
@@ -192,8 +191,7 @@ async def get_warnings(run_id, user: User = Depends(authenticate)):
     if data is None or data == []:
         return Response(status_code=204)
 
-    escaped_data = [html_escape_multi(note) for note in data]
-    return ORJSONResponseObjKeep({'success': True, 'data': escaped_data})
+    return ORJSONResponseObjKeep({'success': True, 'data': data})
 
 
 @router.get('/v1/network/{run_id}')
@@ -213,8 +211,7 @@ async def get_network(run_id, user: User = Depends(authenticate)):
     params = (user.is_super_user(), user.visible_users(), run_id)
     data = DB().fetch_all(query, params=params)
 
-    escaped_data = html_escape_multi(data)
-    return ORJSONResponseObjKeep({'success': True, 'data': escaped_data})
+    return ORJSONResponseObjKeep({'success': True, 'data': data})
 
 
 @router.get('/v1/repositories')
@@ -262,9 +259,7 @@ async def get_repositories(uri: str | None = None, branch: str | None = None, ma
     if data is None or data == []:
         return Response(status_code=204) # No-Content
 
-    escaped_data = [html_escape_multi(run) for run in data]
-
-    return ORJSONResponse({'success': True, 'data': escaped_data})
+    return ORJSONResponse({'success': True, 'data': data})
 
 
 @router.get('/v1/runs', deprecated=True)
@@ -330,9 +325,7 @@ async def get_runs(uri: str | None = None, branch: str | None = None, machine_id
     if data is None or data == []:
         return Response(status_code=204) # No-Content
 
-    escaped_data = [html_escape_multi(run) for run in data]
-
-    return ORJSONResponse({'success': True, 'data': escaped_data})
+    return ORJSONResponse({'success': True, 'data': data})
 
 
 # Just copy and paste if we want to deprecate URLs
@@ -714,8 +707,6 @@ async def get_watchlist(user: User = Depends(authenticate)):
 @router.post('/v1/software/add')
 async def software_add(software: Software, user: User = Depends(authenticate)):
 
-    software = html_escape_multi(software)
-
     if software.name is None or software.name.strip() == '':
         raise RequestValidationError('Name is empty')
 
@@ -799,8 +790,6 @@ async def get_run(run_id: str, user: User = Depends(authenticate)):
     if data is None or data == []:
         return Response(status_code=204) # No-Content
 
-    data = html_escape_multi(data)
-
     return ORJSONResponseObjKeep({'success': True, 'data': data})
 
 @router.get('/v1/optimizations/{run_id}')

frontend/js/ci-index.js

@@ -10,20 +10,20 @@ async function getRepositories(sort_by = 'date') {
     table_body.innerHTML = '';
 
     api_data.data.forEach(el => {
-        const repo = el[0]; // escaping not needed, as done in API ingest
-        const source = el[1]; // escaping not needed, as done in API ingest
-        const last_run = el[2]; // escaping not needed, as done in API ingest
+        const repo = el[0];
+        const source = el[1];
+        const last_run = el[2];
 
         let row = table_body.insertRow()
         row.innerHTML = `
             <td>
                 <div class="ui accordion" style="width: 100%;">
                   <div class="title">
-                    <i class="dropdown icon"></i> ${getRepoLink(repo, source)}
+                    <i class="dropdown icon"></i> ${getRepoLink(repo, source)} <!-- raw values: function handles escaping internally -->
                     <span class="ui label right icon" style="float: right;">${dateToYMD(new Date(last_run), short=true)}<i class="clock icon"></i></span>
                   </div>
 
-                  <div class="content" data-uri="${repo}">
+                  <div class="content" data-uri="${escapeString(repo)}">
                       <table class="ui celled striped table"></table>
                   </div>
                 </div>
@@ -53,18 +53,17 @@ function getRepoLink(repo, source) {
         iconClass = 'bitbucket';
     }
 
-    // Assumes the repo var is sanitized before being sent to this function
-    return `<i class="icon ${iconClass}"></i>${repo} <a href="${getRepoUri(repo, source)}"><i class="icon external alternate"></i></a>`;
+    return `<i class="icon ${iconClass}"></i>${escapeString(repo)} <a href="${getRepoUri(repo, source)}"><i class="icon external alternate"></i></a>`;
 }
 
 // Function to generate the repository URI
 function getRepoUri(repo, source) {
     if (source.startsWith('github')) {
-        return `https://www.github.com/${repo}`;
+        return `https://www.github.com/${encodeURIComponent(repo)}`;
     } else if (source.startsWith('gitlab')) {
-        return `https://www.gitlab.com/${repo}`;
+        return `https://www.gitlab.com/${encodeURIComponent(repo)}`;
     } else if (source.startsWith('bitbucket')) {
-        return `https://bitbucket.com/${repo}`;
+        return `https://bitbucket.com/${encodeURIComponent(repo)}`;
     }
 }
 
@@ -81,7 +80,7 @@ const getCIRunsTable = async (el, url, include_uri=true, include_button=true, se
     const columns = [
         {
             data: 0, title: 'Workflow', render: function(el,type,row) {
-                return `<a href="/ci.html?repo=${row[0]}&branch=${row[1]}&workflow=${row[2]}">${row[5]}</a>`;
+                return `<a href="/ci.html?repo=${encodeURIComponent(row[0])}&branch=${encodeURIComponent(row[1])}&workflow=${encodeURIComponent(row[2])}">${escapeString(row[5])}</a>`;
             }
         },
         {data : 1, title: 'Branch'},
@@ -94,7 +93,7 @@ const getCIRunsTable = async (el, url, include_uri=true, include_button=true, se
         },
         {
             title: 'Carbon', render: function(el,type,row) {
-                return `<img src="${API_URL}/v1/ci/badge/get?repo=${row[0]}&branch=${row[1]}&workflow=${row[2]}&mode=totals&metric=carbon&duration_days=30" onerror="this.src='/images/no-data-badge.webp'">`;
+                return `<img src="${API_URL}/v1/ci/badge/get?repo=${encodeURIComponent(row[0])}&branch=${encodeURIComponent(row[1])}&workflow=${encodeURIComponent(row[2])}&mode=totals&metric=carbon&duration_days=30" onerror="this.src='/images/no-data-badge.webp'">`;
             }
         },
 

frontend/js/ci.js

@@ -154,7 +154,7 @@ const displayStatsTable = (stats) => {
         const label_stats_avg_node = document.createElement("tr")
         const label = row[21];
         label_stats_avg_node.innerHTML += `
-                                        <td class="td-index" data-tooltip="Averages per step '${label}'"  data-position="top left">${label} <i class="question circle icon small"></i></td>
+                                        <td class="td-index" data-tooltip="Averages per step '${escapeString(label)}'"  data-position="top left">${escapeString(label)} <i class="question circle icon small"></i></td>
                                         <td class=" td-index">${numberFormatter.format(row[0]/1000000)} J (± ${numberFormatter.format(row[3])}%)</td>
                                         <td class=" td-index">${numberFormatter.format(row[4]/1000000)} s (± ${numberFormatter.format(row[7])}%)</td>
                                         <td class=" td-index">${numberFormatter.format(row[8])}% (± ${numberFormatter.format(row[11])}%%)</td>
@@ -166,7 +166,7 @@ const displayStatsTable = (stats) => {
 
         const label_stats_total_node = document.createElement("tr")
         label_stats_total_node.innerHTML += `
-                                        <td class="td-index" data-tooltip="Totals per step '${label}'"  data-position="top left">${label} <i class="question circle icon small"></i></td>
+                                        <td class="td-index" data-tooltip="Totals per step '${escapeString(label)}'"  data-position="top left">${escapeString(label)} <i class="question circle icon small"></i></td>
                                         <td class=" td-index">${numberFormatter.format(row[0]/1000000)} J (± ${numberFormatter.format(row[3])}%)</td>
                                         <td class=" td-index">${numberFormatter.format(row[4]/1000000)} s (± ${numberFormatter.format(row[7])}%)</td>
                                         <td class=" td-index">${numberFormatterLong.format(row[17]/1000000)} gCO2e (± ${numberFormatter.format(row[19])}%)</td>
@@ -193,13 +193,13 @@ const displayRunDetailsTable = (measurements, repo) => {
 
         let run_link = '';
 
-        const run_id_esc = escapeString(run_id)
+        const run_id_esc = encodeURIComponent(run_id)
 
         if(source == 'github') {
-            run_link = `https://github.com/${repo}/actions/runs/${run_id_esc}`;
+            run_link = `https://github.com/${encodeURIComponent(repo)}/actions/runs/${run_id_esc}`;
         }
         else if (source == 'gitlab') {
-            run_link = `https://gitlab.com/${repo}/-/pipelines/${run_id_esc}`
+            run_link = `https://gitlab.com/${encodeURIComponent(repo)}/-/pipelines/${run_id_esc}`
         }
 
         const run_link_node = `<a href="${run_link}" target="_blank">${run_id_esc}</a>`
@@ -216,7 +216,7 @@ const displayRunDetailsTable = (measurements, repo) => {
                             <td class="td-index">${escapeString(cpu)}</td>\
                             <td class="td-index">${escapeString(cpu_avg)}%</td>
                             <td class="td-index">${numberFormatter.format(duration_us/1000000)} s</td>
-                            <td class="td-index" ${escapeString(tooltip)}>${escapeString(short_hash)}</td>\
+                            <td class="td-index" title="${escapeString(commit_hash)}">${escapeString(short_hash)}</td>\
                             <td class="td-index">${city_string}</td>
                             <td class="td-index">${escapeString(carbon_intensity_g)} gCO2/kWh</td>
                             <td class="td-index" title="${carbon_ug/1000000}">${escapeString(numberFormatterLong.format(carbon_ug/1000000))} gCO2e</td>
@@ -232,9 +232,9 @@ const getBadges = async (repo, branch, workflow_id) => {
     try {
         const link_node = document.createElement("a")
         const img_node = document.createElement("img")
-        img_node.src = `${API_URL}/v1/ci/badge/get?repo=${repo}&branch=${branch}&workflow=${workflow_id}`
+        img_node.src = `${API_URL}/v1/ci/badge/get?repo=${encodeURIComponent(repo)}&branch=${encodeURIComponent(branch)}&workflow=${encodeURIComponent(workflow_id)}`
         img_node.onerror = function() {this.src='/images/no-data-badge.webp'}
-        link_node.href = `${METRICS_URL}/ci.html?repo=${repo}&branch=${branch}&workflow=${workflow_id}`
+        link_node.href = `${METRICS_URL}/ci.html?repo=${encodeURIComponent(repo)}&branch=${encodeURIComponent(branch)}&workflow=${encodeURIComponent(workflow_id)}`
         link_node.rel = 'noopener'
         link_node.target = '_blank'
 
@@ -290,7 +290,7 @@ const getBadges = async (repo, branch, workflow_id) => {
 
 const getMeasurementsAndStats = async (repo, branch, workflow_id, start_date, end_date) => {
 
-    const query_string=`repo=${repo}&branch=${branch}&workflow=${workflow_id}&start_date=${start_date}&end_date=${end_date}`;
+    const query_string=`repo=${escapeString(repo)}&branch=${escapeString(branch)}&workflow=${escapeString(workflow_id)}&start_date=${start_date}&end_date=${end_date}`;
     const [measurements, stats] = await Promise.all([
         makeAPICall(`/v1/ci/measurements?${query_string}`),
         makeAPICall(`/v1/ci/stats?${query_string}`)
@@ -357,23 +357,23 @@ const refreshView = async (repo, branch, workflow_id, chart_instance) => {
 
 const populateRunInfos = async (repo, branch, source, workflow_name, workflow_id) => {
 
-    document.querySelector('#ci-data-branch').innerText =  branch;
-    document.querySelector('#ci-data-workflow-id').innerText =  workflow_id;
+    document.querySelector('#ci-data-branch').innerText =  escapeString(branch);
+    document.querySelector('#ci-data-workflow-id').innerText =  escapeString(workflow_id);
 
     if (workflow_name == '' || workflow_name == null) {
         workflow_name = workflow_id ;
     }
-    document.querySelector('#ci-data-workflow').innerText =  workflow_name;
+    document.querySelector('#ci-data-workflow').innerText =  escapeString(workflow_name);
 
     let repo_link = ''
     if(source == 'github') {
-        repo_link = `https://github.com/${repo}`;
+        repo_link = `https://github.com/${encodeURIComponent(repo)}`;
     }
     else if(source == 'gitlab') {
-        repo_link = `https://gitlab.com/${repo}`;
+        repo_link = `https://gitlab.com/${encodeURIComponent(repo)}`;
     }
 
-    const repo_link_node = `<a href="${repo_link}" target="_blank">${repo}</a>`
+    const repo_link_node = `<a href="${repo_link}" target="_blank">${escapeString(repo)}</a>`
     document.querySelector('#ci-data-repo').innerHTML =  repo_link_node;
 }