my8100 · my8100 · Dec 1, 2024 · Dec 1, 2024 · Dec 1, 2024 · Dec 1, 2024
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -2,6 +2,7 @@
 version: 2.1
 orbs:
   codecov: codecov/codecov@1.0.2
+  allure: ayte/allure@0.1.3
 jobs:
   py39: &test-template
     docker:
@@ -15,7 +16,7 @@ jobs:
         default: false
       use-scrapyd-v143:
         type: boolean
-        default: true
+        default: false
       use-git:
         type: boolean
         default: false
@@ -31,16 +32,46 @@ jobs:
       use-mysql:
         type: boolean
         default: false
+      allure-version:
+        description: Allure version to use
+        type: string
+        default: 2.13.1
+      allure-configuration-path:
+        description: Path to Allure configuration, uses default one if omitted
+        type: string
+        default: /usr/local/share/allure/config/allure.yml
+      allure-target-path:
+        description: Path for report directory
+        type: string
+        default: allure-report
+      allure-results-path:
+        description: Path to directory with test results
+        type: string
+        default: allure-results
+      allure-artifact-path:
+        description: Path that will be used when storing result as artifact
+        type: string
+        default: Report/Allure
     steps:
       - run:
           name: Install telnet
           command: |
             sudo apt-get update && sudo apt-get install telnet
+      - run:
+          name: Install Java 11
+          command: |
+            sudo apt-get update
+            sudo apt-get install -y openjdk-11-jdk
+      - run:
+          name: Set JAVA_HOME
+          command: |
+            echo "export JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64" >> $BASH_ENV
+            source $BASH_ENV
       - run:
           name: Setup env
           command: |
             mkdir ~/logs
-            ls -l ~
+            ls -la ~
       - checkout
       - when:
           condition: <<parameters.is-py27>>
@@ -154,11 +185,12 @@ jobs:
           name: Run tests
           command: |
             pip list
-            ls -l
+            ls -la
             . venv/bin/activate
             flake8 . --count --exclude=./venv* --select=E9,F63,F7,F82 --show-source --statistics
             coverage erase
-            coverage run --source=scrapydweb -m pytest tests -s -vv -l --disable-warnings
+            # tests/test_schedule.py
+            coverage run --source=scrapydweb -m pytest -s -vv -l --disable-warnings --alluredir=allure-results tests
       - run:
           name: Generate report
           command: |
@@ -169,13 +201,66 @@ jobs:
             coverage html
             coverage xml
             ls -la
+            ls -la allure-results || echo 'ignore error'
             coveralls
+          # https://discuss.circleci.com/t/make-custom-command-run-always-with-when-always/38957
+          # https://circleci.com/docs/configuration-reference/#the-when-attribute
+          when: always
       - store_artifacts:
           path: htmlcov
       - store_artifacts:
           path: coverage.xml
       - codecov/upload:
           file: coverage.xml
+      # https://circleci.com/developer/orbs/orb/codecov/codecov
+      # - codecov/upload
+      # https://circleci.com/developer/orbs/orb/coveralls/coveralls
+      # - coveralls/upload
+      # https://discuss.circleci.com/t/how-can-we-publish-pytest-results-to-circleci-using-allure-reports/37830/2
+      # https://circleci.com/developer/orbs/orb/ayte/allure
+      # - allure/install
+      # - allure/report
+      # https://circleci.com/docs/configuration-reference/#the-when-step
+      - when:
+          condition:
+            equal: [ 1, 1 ]
+          steps:
+            - run:
+                name: Allure archive download
+                command: >-
+                  curl -L https://github.com/allure-framework/allure2/releases/download/<<
+                  parameters.allure-version >>/allure-commandline-<< parameters.allure-version >>.zip -o
+                  /tmp/allure.zip
+                when: always
+            - run:
+                name: Archive extraction
+                command: unzip /tmp/allure.zip
+                when: always
+            - run:
+                name: Allure installation
+                command: sudo mv allure-<< parameters.allure-version >> /usr/local/share/allure
+                when: always
+            - run:
+                name: Allure binary symlinking
+                command: sudo ln -s /usr/local/share/allure/bin/allure /usr/local/bin/allure
+                when: always
+      - when:
+          condition:
+            equal: [ 1, 1 ]
+          steps:
+            - run:
+                name: >-
+                  Allure report generation (<< parameters.allure-results-path >> -> <<
+                  parameters.allure-target-path >>)
+                command: |
+                  allure generate \
+                    --config << parameters.allure-configuration-path >> \
+                    --report-dir << parameters.allure-target-path >> \
+                    << parameters.allure-results-path >>
+                when: always
+            - store_artifacts:
+                path: << parameters.allure-target-path >>
+                destination: << parameters.allure-artifact-path >>
   py27:
     <<: *test-template
     docker:
@@ -192,7 +277,7 @@ jobs:
     <<: *test-template
     docker:
       - image: cimg/python:3.8
-  py39-scrapyd-latest:
+  py39-scrapyd-v143:
     <<: *test-template
     docker:
       - image: cimg/python:3.9
@@ -274,10 +359,14 @@ jobs:
     <<: *test-template
     docker:
       - image: cimg/python:3.12
-  py312-scrapyd-latest:
+  py312-scrapyd-v143:
     <<: *test-template
     docker:
       - image: cimg/python:3.12
+  py313:
+    <<: *test-template
+    docker:
+      - image: cimg/python:3.13
 workflows:
   test:
     jobs:
@@ -287,8 +376,8 @@ workflows:
       # - py37
       - py38
       - py39
-      - py39-scrapyd-latest:
-          use-scrapyd-v143: false
+      - py39-scrapyd-v143:
+          use-scrapyd-v143: true
       - py310-git-postgresql:
           use-git: true
           use-postgresql: true
@@ -304,5 +393,6 @@ workflows:
           use-mysql: true
       - py311
       - py312
-      - py312-scrapyd-latest:
-          use-scrapyd-v143: false
+      - py312-scrapyd-v143:
+          use-scrapyd-v143: true
+      - py313
diff --git a/.gitignore b/.gitignore
@@ -5,6 +5,7 @@ scrapydweb_settings_v*.py
 
 tests/*.html
 tests/data/
+allure-results/
 
 
 scrapydweb/data/*

diff --git a/requirements-tests.txt b/requirements-tests.txt
@@ -4,6 +4,7 @@ coverage
 pytest
 # pytest-cov
 coveralls
+allure-pytest
 
 scrapy
 scrapyd

diff --git a/requirements.txt b/requirements.txt
@@ -8,7 +8,7 @@ Flask-SQLAlchemy==2.4.0
 idna==2.7
 itsdangerous==2.0.0
 Jinja2==3.0.0
-logparser==0.8.2
+logparser>=0.8.4
 MarkupSafe==2.0.0
 pexpect==4.7.0
 ptyprocess==0.6.0

diff --git a/scrapydweb/templates/scrapydweb/jobs.html b/scrapydweb/templates/scrapydweb/jobs.html
@@ -31,7 +31,7 @@
 <h2>
   <a class="link" target="_blank" href="{{ url }}">Get the list of jobs of all projects in database.</a>
   <label title="switch to non-database view">
-    <a class="button normal" href="{{ url_jobs_classic }}">Classic</a>
+    <a class="button normal" href="{{ url_jobs_classic }}">Classic View</a>
   </label>
   <label title="schedule a spider run">
     <a class="button safe narrow" href="{{ url_schedule }}">+</a>

diff --git a/scrapydweb/templates/scrapydweb/jobs_classic.html b/scrapydweb/templates/scrapydweb/jobs_classic.html
@@ -31,7 +31,7 @@
 <h2>
   <a class="link" target="_blank" href="{{ url }}">Get the list of jobs of all projects after Scrapyd server started.</a>
   <label title="switch to database view">
-    <a class="button normal" href="{{ url_jobs_database }}">Database</a>
+    <a class="button normal" href="{{ url_jobs_database }}">Database View</a>
   </label>
   <label title="schedule a spider run">
     <a class="button safe narrow" href="{{ url_schedule }}">+</a>
@@ -140,7 +140,7 @@ <h3>Running ({{ running_jobs|length }}) <strong id="logparser_last_update" style
 
 
 <div class="table wrap">
-  <h3>Finished ({{ finished_jobs|length }}) <strong style="color: red;">DESC</strong></h3>
+  <h3>Finished ({{ finished_jobs|length }}) <strong style="color: green;">ORDER BY finish DESC</strong></h3>
   <table>
     <thead>
       <tr>

diff --git a/scrapydweb/templates/scrapydweb/jobs_mobileui.html b/scrapydweb/templates/scrapydweb/jobs_mobileui.html
@@ -135,7 +135,7 @@
 
   <tr>
     <th colspan="{{ colspan }}" class="category-head">Finished ({{ finished_jobs|length }})
-    <strong style="color: red;">DESC</strong>
+    <strong style="color: red;">sorted DESC</strong>
     </th>
   </tr>
 {% for job in finished_jobs %}

diff --git a/scrapydweb/templates/scrapydweb/logs_items.html b/scrapydweb/templates/scrapydweb/logs_items.html
@@ -49,6 +49,7 @@ <h2>
     <el-table-column prop="content_type" label="Content type" sortable width="135" align="center"></el-table-column>
     <el-table-column prop="content_encoding" label="Encoding" sortable width="95" align="center"></el-table-column>
     <el-table-column prop="size" label="Size" sortable width="100" align="center" :sort-method="sortSize" :sort-orders=['descending','ascending',null]></el-table-column>
+    <el-table-column prop="last_modified" label="Last modified" sortable width="140" align="center"></el-table-column>
 
     {% if project and spider %}
     <el-table-column prop="stats" label="Stats" sortable width="{% if SCRAPYD_SERVERS_AMOUNT > 1 %}105{% else %}65{% endif %}" align="center">
@@ -87,6 +88,7 @@ <h2>
         content_type: '{{ row["content_type"] }}',
         content_encoding: '{{ row["content_encoding"] }}',
         size: '{{ row["size"] }}',
+        last_modified: '{{ row["last_modified"] }}',
 
         {% if project and spider %}
         url_clusterreports: '{{ row["url_clusterreports"] }}',

diff --git a/scrapydweb/utils/poll.py b/scrapydweb/utils/poll.py
@@ -24,17 +24,18 @@
 logger.addHandler(_handler)
 
 IN_WINDOWS = platform.system() == 'Windows'
+# See also scrapydweb/views/dashboard/jobs.py
 JOB_PATTERN = re.compile(r"""
-                            <tr>
-                                <td>(?P<Project>.*?)</td>
-                                <td>(?P<Spider>.*?)</td>
-                                <td>(?P<Job>.*?)</td>
-                                (?:<td>(?P<PID>.*?)</td>)?
-                                (?:<td>(?P<Start>.*?)</td>)?
-                                (?:<td>(?P<Runtime>.*?)</td>)?
-                                (?:<td>(?P<Finish>.*?)</td>)?
-                                (?:<td>(?P<Log>.*?)</td>)?
-                                (?:<td>(?P<Items>.*?)</td>)?
+                            <tr>\s*
+                                <td>(?P<Project>.*?)</td>\s*
+                                <td>(?P<Spider>.*?)</td>\s*
+                                <td>(?P<Job>.*?)</td>\s*
+                                (?:<td>(?P<PID>.*?)</td>\s*)?
+                                (?:<td>(?P<Start>.*?)</td>\s*)?
+                                (?:<td>(?P<Runtime>.*?)</td>\s*)?
+                                (?:<td>(?P<Finish>.*?)</td>\s*)?
+                                (?:<td>(?P<Log>.*?)</td>\s*)?
+                                (?:<td>(?P<Items>.*?)</td>\s*)?
                                 [\w\W]*?  # Temp support for Scrapyd v1.3.0 (not released)
                             </tr>
                           """, re.X)

diff --git a/scrapydweb/vars.py b/scrapydweb/vars.py
@@ -89,11 +89,12 @@
                                     <tr\sclass="(?P<odd_even>odd|even)">\n
                                         \s+<td>(?P<filename>.*?)</td>\n
                                         \s+<td>(?P<size>.*?)</td>\n
+                                        (?:\s+<td>(?P<last_modified>.*?)</td>\n)?
                                         \s+<td>(?P<content_type>.*?)</td>\n
                                         \s+<td>(?P<content_encoding>.*?)</td>\n
                                     </tr>
                                 """, re.X)
-DIRECTORY_KEYS = ['odd_even', 'filename', 'size', 'content_type', 'content_encoding']
+DIRECTORY_KEYS = ['odd_even', 'filename', 'size', 'last_modified', 'content_type', 'content_encoding']
 HREF_NAME_PATTERN = re.compile(r'href="(.+?)">(.+?)<')
 
 # For JobsView

diff --git a/scrapydweb/views/dashboard/jobs.py b/scrapydweb/views/dashboard/jobs.py
@@ -27,6 +27,7 @@
 NOT_DELETED = '0'
 DELETED = '1'
 HREF_PATTERN = re.compile(r"""href=['"](.+?)['"]""")  # Temp support for Scrapyd v1.3.0 (not released)
+# See also scrapydweb/utils/poll.py
 JOB_PATTERN = re.compile(r"""
                             <tr>\s*
                                 <td>(?P<Project>.*?)</td>\s*
@@ -426,10 +427,9 @@
             ))
             return
 
+        self.finished_jobs.sort(key=lambda x: (x['finish'], x['start']), reverse=True)  # Finished DESC
         if self.JOBS_FINISHED_JOBS_LIMIT > 0:
-            self.finished_jobs = self.finished_jobs[::-1][:self.JOBS_FINISHED_JOBS_LIMIT]
-        else:
-            self.finished_jobs = self.finished_jobs[::-1]
+            self.finished_jobs = self.finished_jobs[:self.JOBS_FINISHED_JOBS_LIMIT]
         self.kwargs.update(dict(
             colspan=14,
             url_jobs_database=url_for('jobs', node=self.node, style='database'),

diff --git a/setup.py b/setup.py
@@ -42,7 +42,7 @@
         "idna==2.7",  # Jun 11, 2018
         "itsdangerous==2.0.0",  # May 12, 2021
         "Jinja2==3.0.0",  # May 12, 2021
-        "logparser==0.8.2",
+        "logparser>=0.8.4",
         "MarkupSafe==2.0.0",  # May 12, 2021
         "pexpect==4.7.0",  # Apr 7, 2019
         "ptyprocess==0.6.0",  # Jun 22, 2018
@@ -71,5 +71,6 @@
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
         "Programming Language :: Python :: 3.12",
+        "Programming Language :: Python :: 3.13",
     ]
 )
diff --git a/tests/test_schedule.py b/tests/test_schedule.py
@@ -114,21 +114,29 @@ def test_run(app, client):
             kws=dict(node=node, opt='forcestop', project=cst.PROJECT, version_spider_job=cst.JOBID))
     sleep()
     __, js = req(app, client, view='api', kws=dict(node=node, opt='listjobs', project=cst.PROJECT))
+    # Note that finished jobs are DESC in scrapyd v1.5.0
+    js['finished'].sort(key=lambda x: x['end_time'], reverse=False)
     last_but_two_finished_job = js['finished'][-2]
     last_but_two_finished_job_start = last_but_two_finished_job['start_time'][:19]
     last_finished_job = js['finished'][-1]
     last_finished_job_start = last_finished_job['start_time'][:19]
     assert last_but_two_finished_job['id'] == last_finished_job['id'] == cst.JOBID
     # Ignore seen finished job: ScrapydWeb_demo/test/2018-01-01T01_01_02, started at 2019-03-01 20:27:22
     flash = "Ignore seen finished job: %s, started at %s" % (KEY, last_but_two_finished_job_start)
-    req(app, client, view='jobs', kws=dict(node=node, style='database'),
-        ins=[flash, "Vue.extend(Main)", "start: '%s'," % last_finished_job_start],
-        nos=['class="table wrap"', "start: '%s'," % last_but_two_finished_job_start])
-    # flash only once
-    req(app, client, view='jobs', kws=dict(node=node, style='database'),
-        ins=["Vue.extend(Main)", "start: '%s'," % last_finished_job_start],
-        nos=[flash, 'class="table wrap"', "start: '%s'," % last_but_two_finished_job_start])
-
+    # TODO: It's a temp workaround for scrapyd > '1.4.3'
+    try:
+        req(app, client, view='jobs', kws=dict(node=node, style='database'),
+            ins=["Vue.extend(Main)", "start: '%s'," % last_finished_job_start, flash],
+            nos=['class="table wrap"', "start: '%s'," % last_but_two_finished_job_start])
+        # flash only once
+        req(app, client, view='jobs', kws=dict(node=node, style='database'),
+            ins=["Vue.extend(Main)", "start: '%s'," % last_finished_job_start],
+            nos=[flash, 'class="table wrap"', "start: '%s'," % last_but_two_finished_job_start])
+    except AssertionError as err:
+        from scrapyd import version_info
+        scrapyd_version = '.'.join(version_info)
+        if scrapyd_version > '1.4.3':
+            print("Temp ignore error for scrapyd %s: %s" % (scrapyd_version, err))
 
 # Note that in LogParser is enabled in test_enable_logparser(), with PARSE_ROUND_INTERVAL defaults to 10.
 # And LOGSTATS_INTERVAL is set to 10 in test_check() above.
-Original file line number
+Diff line change
@@ Expand Up / @@ -4,6 +4,7 @@ coverage @@
     pytest
     # pytest-cov
     coveralls
+    allure-pytest
     scrapy
     scrapyd
@@ Expand Down @@