From 4b35392d3104ae7e553672a6829ed784ca1cb175 Mon Sep 17 00:00:00 2001 From: BJ Fulton Date: Wed, 20 May 2026 09:59:13 -0600 Subject: [PATCH 01/31] Fix SQLite schema-name lookup in TableValidator and dataDictionary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For SQLite configs, connectInfo['tap_schema'] is the FILE PATH (passed to ATTACH DATABASE), not the schema name. The schema name used in queries is held in connectInfo['tap_schema_file'] (set as ATTACH AS in tapquery.py). TableValidator and dataDictionary both used connectInfo['tap_schema'] as the schema name when constructing SQL like: SELECT table_name FROM .tables which produced "near "/": syntax error" against SQLite because the generated SQL embedded the file path. Surfaced by the new CI SQLite fixture; would have prevented any SQLite- backed production deployment from passing table validation. Oracle/ Postgres/MySQL behavior is unchanged — those configs don't set tap_schema_file, so the existing tap_schema lookup remains the fallback. Co-Authored-By: Claude Opus 4.7 (1M context) --- TAP/datadictionary.py | 29 ++++++++++++++++++----------- TAP/tablevalidator.py | 10 +++++++++- 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/TAP/datadictionary.py b/TAP/datadictionary.py index 3f556d7..7d27719 100644 --- a/TAP/datadictionary.py +++ b/TAP/datadictionary.py @@ -3,9 +3,10 @@ # https://github.com/Caltech-IPAC/nexsciTAP/blob/master/LICENSE -import os import logging +import os import re + from astropy.io import ascii @@ -92,8 +93,8 @@ def __init__(self, conn, table, connectInfo, **kwargs): if self.debug: logging.debug(f'dbtable = {self.dbtable:s}') - logging.debug(f'ddtbl = ' + str(self.ddtbl)) - logging.debug(f'ddfile = ' + str(self.ddfile)) + logging.debug('ddtbl = ' + str(self.ddtbl)) + logging.debug('ddfile = ' + str(self.ddfile)) # @@ -111,20 +112,20 @@ def __init__(self, conn, table, connectInfo, **kwargs): if self.debug: logging.debug('colnames: ' + str(tcolnames)) - i = 0 + i = 0 for trow in tdata: - + dbname = '' desc = '' if 'name' in tcolnames: dbname = trow['name'].lower() desc = dbname - + type = '' if 'intype' in tcolnames: type = trow['intype'] - + unit = '' if 'units' in tcolnames: unit = trow['units'] @@ -147,7 +148,7 @@ def __init__(self, conn, table, connectInfo, **kwargs): if len(dbname) > width: width = len(dbname) - + self.colname[i] = dbname i = i+1 @@ -182,7 +183,13 @@ def __init__(self, conn, table, connectInfo, **kwargs): else: placeholder = '?' - sql = "select * from " + self.connectInfo["tap_schema"] + "." + self.connectInfo["columns_table"] + " where lower(table_name) = " \ + # For SQLite the schema is ATTACHed under a name held in + # `tap_schema_file`; `tap_schema` is the file path. For + # Oracle/Postgres/MySQL `tap_schema` is the schema name itself. + schema_name = self.connectInfo.get("tap_schema_file") \ + or self.connectInfo["tap_schema"] + + sql = "select * from " + schema_name + "." + self.connectInfo["columns_table"] + " where lower(table_name) = " \ + placeholder if self.debug: @@ -302,7 +309,7 @@ def __init__(self, conn, table, connectInfo, **kwargs): # # { while loop # - + #rows = cursor.fetchmany(self.nfetch) rows = cursor.fetchall() @@ -314,7 +321,7 @@ def __init__(self, conn, table, connectInfo, **kwargs): i = 0 for row in rows: - + # # { for loop: each row in the file represents # a column in data dictionary diff --git a/TAP/tablevalidator.py b/TAP/tablevalidator.py index e2d25ca..74236b7 100644 --- a/TAP/tablevalidator.py +++ b/TAP/tablevalidator.py @@ -32,7 +32,15 @@ def __init__(self, conn, connectInfo=None, debug=0): self.tables_table = 'tables' if connectInfo is not None: - if 'tap_schema' in connectInfo: + # `tap_schema_file` is the ATTACH-AS name for SQLite ATTACHed + # schemas (set in tapquery.py:395). For Oracle/Postgres/MySQL + # there's no ATTACH and `tap_schema` is the schema name itself. + # For SQLite, `tap_schema` is the FILE PATH and using it directly + # in a `SELECT ... FROM .tables` reference produces a SQL + # syntax error. Prefer the more-specific key when it's set. + if 'tap_schema_file' in connectInfo: + self.tap_schema = connectInfo['tap_schema_file'] + elif 'tap_schema' in connectInfo: self.tap_schema = connectInfo['tap_schema'] if 'tables_table' in connectInfo: self.tables_table = connectInfo['tables_table'] From b2e2b7866d960c1866b78c92ed3a92a5f0cd6e20 Mon Sep 17 00:00:00 2001 From: BJ Fulton Date: Wed, 20 May 2026 09:59:39 -0600 Subject: [PATCH 02/31] Add CI: GitHub Actions, SQLite test fixture, ruff lint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 1+2 of the CI plan. Establishes a GitHub Actions gate that runs on every PR and on push to develop/main, replacing the manual KOA/NEID/NEA testing loop with automated checks. What runs in CI: - ruff (E, F, W, I) — gated; codebase passes with the minimal ignore set documented in pyproject.toml - pytest tests/ — matrix over Python 3.9 / 3.10 / 3.11 / 3.12 on Ubuntu - python -m build --wheel — Ubuntu + macOS, verifies the C extension builds cross-platform and the wheel is installable Test infrastructure: - tests/conftest.py spins up an HTTP server on a free port backed by a custom nph-aware request handler (stdlib's CGIHTTPRequestHandler prepends its own status line, breaking nph- scripts that emit their own HTTP/1.1 status lines like nph-tap.py does) - tests/fixtures/build_test_db.py builds tap_schema.db (IVOA-shape) and test_data.db (3 public + 3 expired + 4 still-embargoed L1 rows). The still-embargoed rows are the key fixture: production test servers (vmneidtest, koa-test) have all-expired propints, which means propflag enforcement can't be observed there. CI now has the missing scenario. - tests/fixtures/stubs/spatial_index.py is a minimal stub for the four module-level constants TAP imports. The real spatial_index package on PyPI only ships wheels for Python ≤ 3.8, so modern-Python CI cannot install it. Tracking as upstream issue. - tests/test_http_endpoint.py is a smoke layer that exercises the full HTTP→CGI→TAP→SQLite→IPAC pipeline. Security/regression tests land in the follow-up PR (Phase 3+4). Cosmetic ruff autofixes touched 13 files (whitespace, f-strings, import order) — no semantic changes. Deferred to follow-up PRs: - F821 ignored in ruff for now; 6 pre-existing undefined-name bugs to address in TAP/taputil.py (SQLite/MySQL connection branches) and TAP/tap.py:2402 (debug-log typo `result` → `results`) - spatial_index Python 3.9+ wheel situation upstream - Postgres/MySQL/Oracle CI matrix (SQLite-only for first cut) Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/ci.yml | 60 +++ .gitignore | 4 +- TAP/configparam.py | 53 +-- TAP/propfilter.py | 124 ++++--- TAP/tablenames.py | 6 +- TAP/tap.py | 505 +++++++++++++------------- TAP/tapquery.py | 114 +++--- TAP/taputil.py | 33 +- TAP/vositables.py | 297 ++++++++------- TAP/writeresult.py | 177 +++++---- ctest/ctap.py | 13 +- ctest/dbtap.py | 20 +- load_schema/load_schema.py | 6 +- pyproject.toml | 32 ++ requirements-test.txt | 4 + setup.py | 16 +- tests/_helpers.py | 22 ++ tests/conftest.py | 202 +++++++++++ tests/fixtures/TAP.conf.template | 45 +++ tests/fixtures/build_test_db.py | 344 ++++++++++++++++++ tests/fixtures/nph-tap.py | 10 + tests/fixtures/stubs/spatial_index.py | 22 ++ tests/test_http_endpoint.py | 89 +++++ tests/test_tablevalidator.py | 2 +- 24 files changed, 1502 insertions(+), 698 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 pyproject.toml create mode 100644 requirements-test.txt create mode 100644 tests/_helpers.py create mode 100644 tests/conftest.py create mode 100644 tests/fixtures/TAP.conf.template create mode 100644 tests/fixtures/build_test_db.py create mode 100644 tests/fixtures/nph-tap.py create mode 100644 tests/fixtures/stubs/spatial_index.py create mode 100644 tests/test_http_endpoint.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..0e5a578 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,60 @@ +name: CI + +on: + pull_request: + push: + branches: [main, develop] + +jobs: + test: + name: test (py${{ matrix.python-version }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.9", "3.10", "3.11", "3.12"] + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install package and test deps + run: | + pip install --upgrade pip + pip install -r requirements-test.txt + pip install . + + - name: Lint with ruff + run: ruff check . + + - name: Run tests + run: pytest tests/ -v + + wheel-platforms: + name: wheel build (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + steps: + - uses: actions/checkout@v4 + + - name: Set up Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Build wheel + run: | + pip install --upgrade pip build + python -m build --wheel + + - name: Verify wheel artifact + run: | + ls dist/ + python -m pip install dist/*.whl + python -c "import TAP; print('TAP imports OK')" diff --git a/.gitignore b/.gitignore index fcd07d6..035818a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,13 +1,15 @@ *.[oa] *~ +*.so /Sphinx/_build /TAP/__pycache__ -/TAP/writerecs.cpython-36m-darwin.so /build /dist /final_dist /final_wheel /nexsciTAP.egg-info +__pycache__/ +.pytest_cache/ # Workflow artifacts (Claude Code planning/spec docs, not project documentation) /docs/superpowers/ diff --git a/TAP/configparam.py b/TAP/configparam.py index 5f533b7..58701e8 100644 --- a/TAP/configparam.py +++ b/TAP/configparam.py @@ -3,11 +3,12 @@ # https://github.com/Caltech-IPAC/nexsciTAP/blob/master/LICENSE -import os import logging -import configobj +import os import pprint +import configobj + class configParam: @@ -31,7 +32,7 @@ class configParam: def __init__(self, path, **kwargs): - pp = pprint.PrettyPrinter(indent=3) + pprint.PrettyPrinter(indent=3) @@ -80,7 +81,7 @@ def __init__(self, path, **kwargs): logging.debug('') logging.debug('ConfigObj instantiated successfully') - + ### Web server config parameters ############ self.web = 'WEB' @@ -120,7 +121,7 @@ def __init__(self, path, **kwargs): # HTTP_URL (includingt HTTP_PORT) - self.httpurl = None + self.httpurl = None if ('HTTP_URL' in confobj[self.web]): self.httpurl = confobj[self.web]['HTTP_URL'] @@ -129,7 +130,7 @@ def __init__(self, path, **kwargs): self.msg = 'Failed to find HTTP_URL in config_file' raise Exception(self.msg) - self.port = None + self.port = None if('HTTP_PORT' in confobj[self.web]): self.port = confobj[self.web]['HTTP_PORT'] @@ -139,23 +140,23 @@ def __init__(self, path, **kwargs): # CGIPGM - self.cgipgm = None + self.cgipgm = None if ('CGI_PGM' in confobj[self.web]): self.cgipgm = confobj[self.web]['CGI_PGM'] # ARRAYSIZE - self.arraysize = 10000 + self.arraysize = 10000 if ('ArraySize' in confobj[self.web]): self.arraysize = confobj[self.web]['ArraySize'] # INFOMSG - self.infomsg = '' - if('INFOMSG' in confobj[self.web]): - self.infomsg = confobj[self.web]['INFOMSG'] + self.infomsg = '' + if('INFOMSG' in confobj[self.web]): + self.infomsg = confobj[self.web]['INFOMSG'] if self.debug: logging.debug('') @@ -165,10 +166,10 @@ def __init__(self, path, **kwargs): logging.debug('arraysize = %s', self.infomsg) - + ### Configuration ########################### - # If there was an input configuration 'instance', read 'db_connection' + # If there was an input configuration 'instance', read 'db_connection' # and 'sptind_config' section names from there. Otherwise use the # following defaults: @@ -186,7 +187,7 @@ def __init__(self, path, **kwargs): self.status = 'error' self.msg = 'Failed to find DB_CONNECTION in config_file' raise Exception(self.msg) - + self.sptind_config = None if('SPTIND_CONFIG' in confobj[self.instance]): @@ -222,9 +223,9 @@ def __init__(self, path, **kwargs): self.socket = None self.dbschema = None - self.cookiename = '' - self.accesstbl = '' - self.usertbl = '' + self.cookiename = '' + self.accesstbl = '' + self.usertbl = '' self.propfilter = '' self.fileid = '' self.accessid = '' @@ -247,7 +248,7 @@ def __init__(self, path, **kwargs): if self.debug: logging.debug('') logging.debug('dbms = %s', self.dbms) - + # ORACLE Connection @@ -309,15 +310,15 @@ def __init__(self, path, **kwargs): # KOA Proprietary Access parameters - self.cookiename = '' + self.cookiename = '' if('COOKIENAME' in confobj[self.db_connection]): self.cookiename = confobj[self.db_connection]['COOKIENAME'] - self.accesstbl = '' + self.accesstbl = '' if('ACCESS_TBL' in confobj[self.db_connection]): self.accesstbl = confobj[self.db_connection]['ACCESS_TBL'] - self.usertbl = '' + self.usertbl = '' if('USERS_TBL' in confobj[self.db_connection]): self.usertbl = confobj[self.db_connection]['USERS_TBL'] @@ -428,7 +429,7 @@ def __init__(self, path, **kwargs): self.msg = 'Failed to MySQL DBMS user ID in config_file' raise Exception(self.msg) - + # PASSWORD if ('Password' in confobj[self.db_connection]): @@ -490,7 +491,7 @@ def __init__(self, path, **kwargs): # DATABASE - + if 'DataBase' in confobj[self.db_connection]: self.database = confobj[self.db_connection]['DataBase'] @@ -511,7 +512,7 @@ def __init__(self, path, **kwargs): self.msg = 'Failed to find PostgreSQL username in config_file.' raise Exception(self.msg) - + # PASSWORD if 'Password' in confobj[self.db_connection]: @@ -686,5 +687,5 @@ def __init__(self, path, **kwargs): logging.debug(' accessid = ' + str(self.accessid)) logging.debug(' racol = ' + str(self.racol)) logging.debug(' deccol = ' + str(self.deccol)) - - return + + return diff --git a/TAP/propfilter.py b/TAP/propfilter.py index b73ca38..3a38fc2 100644 --- a/TAP/propfilter.py +++ b/TAP/propfilter.py @@ -3,16 +3,14 @@ # https://github.com/Caltech-IPAC/nexsciTAP/blob/master/LICENSE -import os import logging - -import datetime +import os import time -from TAP.writeresult import writeResult from TAP.datadictionary import dataDictionary from TAP.tablenames import TableNames -from TAP.tablevalidator import TableValidator, TableValidationError +from TAP.tablevalidator import TableValidationError, TableValidator +from TAP.writeresult import writeResult class propFilter: @@ -229,7 +227,7 @@ def __init__(self, **kwargs): logging.debug( 'password = [Not shown for security reasons].') # Change to the following to temporarily debug login - + # logging.debug(f'userid = {self.userid:s}') # logging.debug(f'password = {self.password:s}') @@ -389,7 +387,7 @@ def __init__(self, **kwargs): try: self.maxrec = int(maxrecstr) - except Exception as e: + except Exception: self.msg = "Failed to convert input maxrec value [" + \ maxrecstr + "] to integer." @@ -438,7 +436,7 @@ def __init__(self, **kwargs): logging.debug('Connected to Oracle, database ' + self.dbserver) - except Exception as e: + except Exception: self.status = 'error' self.msg = 'Failed to connect to cx_Oracle' @@ -497,7 +495,7 @@ def __init__(self, **kwargs): logging.debug('') logging.debug('TAP_SCHEMA attached') - except Exception as e: + except Exception: self.status = 'error' self.msg = 'Failed to connect to SQLite3 databases' @@ -749,8 +747,8 @@ def __init__(self, **kwargs): " where " + self.fileid + " in(select " + self.fileid_allowed + \ " from " + self.tmp_fileidAlloweddbtbl + ")" - if(len(self.wherestr) > 0): - sql = sql + ' and (' + self.wherestr[6:] + ')' + if(len(self.wherestr) > 0): + sql = sql + ' and (' + self.wherestr[6:] + ')' if self.debug: logging.debug('') @@ -846,7 +844,7 @@ def __init__(self, **kwargs): coldesc=self.coldesc, racol=self.racol, deccol=self.deccol) - + #deccol=self.deccol, #debug=self.debug) @@ -863,7 +861,7 @@ def __init__(self, **kwargs): if self.debug: logging.debug('') - logging.debug(f'writeResult cursor closed') + logging.debug('writeResult cursor closed') self.outpath = wresult.outpath @@ -871,14 +869,14 @@ def __init__(self, **kwargs): # # Drop all tmp DB tables for oracle because oracle v.12.xxx's - # global temporary table is permanent, not really temporary + # global temporary table is permanent, not really temporary # if (self.dbms.lower() == 'oracle'): - + try: self.conn.close() - + if self.debug: logging.debug ('') logging.debug ('returned from oracle conn.close()') @@ -888,12 +886,12 @@ def __init__(self, **kwargs): if self.debug: logging.debug ('') logging.debug (f'conn.close exception: {str(e):s}') - + # # re-connect # - time.sleep (2.0) - + time.sleep (2.0) + userid = self.connectInfo['userid'] dbserver = self.connectInfo['dbserver'] password = self.connectInfo['password'] @@ -902,11 +900,11 @@ def __init__(self, **kwargs): if self.debug: logging.debug ('') - logging.debug (f'xxx0') + logging.debug ('xxx0') logging.debug (f'userid= {userid:s}') logging.debug (f'password= {password:s}') logging.debug (f'dbserver= {dbserver:s}') - + try: self.conn = cx_Oracle.connect ( \ userid, \ @@ -930,25 +928,25 @@ def __init__(self, **kwargs): logging.debug('') logging.debug('Failed to re-connected to Oracle ') logging.debug(f'e= {str(e):s}') - - pass - + + pass + try: self.__dropDbtbl__(self.tmp_fileidAlloweddbtbl) - + if self.debug: logging.debug('') logging.debug( \ 'returned dropDbtbl: tmp_fileidAlloweddbtbl') except Exception as e: - + if self.debug: logging.debug('') logging.debug( \ f'drop fileidAlloweddbtbl exception: {str(e):s}') pass - + if self.debug: logging.debug('') logging.debug('tmp_fileidAlloweddbtbl dropped') @@ -962,14 +960,14 @@ def __init__(self, **kwargs): try: self.__dropDbtbl__(self.tmp_accessiddbtbl) - + if self.debug: logging.debug('') logging.debug( \ 'returned dropDbtbl: tmp_accessiddbtbl') except Exception as e: - + if self.debug: logging.debug('') logging.debug( \ @@ -1295,8 +1293,8 @@ def __validateUser__(self, cookiename, cookiestr, propfilter, # if self.debug: - logging.debug(f'cookiename = ' + str(cookiename)) - logging.debug(f'cookiestr = ' + str(cookiestr)) + logging.debug('cookiename = ' + str(cookiename)) + logging.debug('cookiestr = ' + str(cookiestr)) msg = '' ind = cookiestr.find(cookiename) @@ -1325,17 +1323,17 @@ def __validateUser__(self, cookiename, cookiestr, propfilter, return arr = substr1.split('|') - narr = len(arr) + len(arr) self.userid = arr[0] self.encodedpass = arr[1] if self.debug: logging.debug('') - logging.debug(f'substr1 = ' + str(substr1)) - logging.debug(f'arr = ' + str(arr)) - logging.debug(f'userid = ' + str(self.userid)) - logging.debug(f'encodedpass = ' + str(self.encodedpass)) + logging.debug('substr1 = ' + str(substr1)) + logging.debug('arr = ' + str(arr)) + logging.debug('userid = ' + str(self.userid)) + logging.debug('encodedpass = ' + str(self.encodedpass)) if(self.userid == 'anon'): self.userid = '' @@ -1397,8 +1395,8 @@ def __validateUser__(self, cookiename, cookiestr, propfilter, if self.debug: logging.debug('') - logging.debug(f'password = [Not shown for security reasons.]') - # Change to following to debug password + logging.debug('password = [Not shown for security reasons.]') + # Change to following to debug password # logging.debug(f'password = {password:s}') if(len(password) == 0): @@ -1589,7 +1587,7 @@ def __createTmpAccessiddb__(self, tmp_accessiddbtbl, userid, accessid, # { createTmpAccessiddb # # - # first drop tmp_accessiddbtbl in case table with the same name + # first drop tmp_accessiddbtbl in case table with the same name # might already exist # @@ -1601,8 +1599,8 @@ def __createTmpAccessiddb__(self, tmp_accessiddbtbl, userid, accessid, if self.debug: logging.debug('') logging.debug('returned dropDbtbl') - logging.debug(f'temp dbtbl dropped') - + logging.debug('temp dbtbl dropped') + except Exception as e: self.msg = 'Failed to create tmp_accessiddbtbl: ' + str(e) @@ -1644,7 +1642,7 @@ def __createTmpAccessiddb__(self, tmp_accessiddbtbl, userid, accessid, logging.debug('') logging.debug('tmp_accessiddbtbl created') - + # Insert into tmp_accessiddbtbl: select accessid allowed # by userid: accessidtbl @@ -1670,8 +1668,8 @@ def __createTmpAccessiddb__(self, tmp_accessiddbtbl, userid, accessid, raise Exception(self.msg) - # check rowcount in tmp_accessiddbtbl - + # check rowcount in tmp_accessiddbtbl + sql = "select * from " + tmp_accessiddbtbl if self.debug: @@ -1715,10 +1713,10 @@ def __createTmpFileiddb__(self, tmp_fileiddbtbl, fileid, fileid_allowed, if self.debug: logging.debug('') - logging.debug(f'Enter createTmpFileiddb') + logging.debug('Enter createTmpFileiddb') if (self.dbms.lower() == 'pgsql'): - + sql = "create temporary table " + tmp_fileiddbtbl + \ "(" + fileid_allowed + " varchar(35)) on commit preserve rows" else: @@ -1753,21 +1751,21 @@ def __createTmpFileiddb__(self, tmp_fileiddbtbl, fileid, fileid_allowed, if(self.propfilter == 'koa'): if(len(self.userid) > 0): - + if (self.dbms.lower() == 'oracle'): access_constraint = \ "((current_date > add_months(date_obs, propint))" + \ " or(lower(" + accessid + ") in(select " + \ accessid + " from " + tmp_accessiddbtbl + ")))" - + elif (self.dbms.lower() == 'pgsql'): access_constraint = "((current_date > " + \ "(date_obs + (propint * '1 month'::interval)))" + \ " or (lower(" + accessid + ") in (select " + \ accessid + " from " + tmp_accessiddbtbl + ")))" - + else: if (self.dbms.lower() == 'oracle'): @@ -1779,7 +1777,7 @@ def __createTmpFileiddb__(self, tmp_fileiddbtbl, fileid, fileid_allowed, access_constraint = "(current_date > " + \ "(date_obs + (propint * '1 month'::interval)))" - + if self.debug: logging.debug('') logging.debug( @@ -1802,20 +1800,20 @@ def __createTmpFileiddb__(self, tmp_fileiddbtbl, fileid, fileid_allowed, "add_months(obsdate, l0propint))" + \ " or(lower(" + accessid + ") in(select " + \ accessid + " from " + tmp_accessiddbtbl + ")))" - + elif (self.dbms.lower() == 'pgsql'): access_constraint = "((current_date > " + \ "(obsdate + (l0propint * '1 month'::interval)))" + \ " or (lower(" + accessid + ") in (select " + \ accessid + " from " + tmp_accessiddbtbl + ")))" - + else: if (self.dbms.lower() == 'oracle'): access_constraint = \ "(current_date > add_months(obsdate, l0propint))" - + elif (self.dbms.lower() == 'pgsql'): access_constraint = "(current_date > " + \ @@ -1832,20 +1830,20 @@ def __createTmpFileiddb__(self, tmp_fileiddbtbl, fileid, fileid_allowed, "add_months(obsdate, l1propint))" + \ " or(lower(" + accessid + ") in(select " + \ accessid + " from " + tmp_accessiddbtbl + ")))" - + elif (self.dbms.lower() == 'pgsql'): access_constraint = "((current_date > " + \ "(obsdate + (l1propint * '1 month'::interval)))" + \ " or (lower(" + accessid + ") in (select " + \ accessid + " from " + tmp_accessiddbtbl + ")))" - + else: if (self.dbms.lower() == 'oracle'): access_constraint = \ "(current_date > add_months(obsdate, l1propint))" - + elif (self.dbms.lower() == 'pgsql'): access_constraint = "(current_date > " + \ @@ -1861,20 +1859,20 @@ def __createTmpFileiddb__(self, tmp_fileiddbtbl, fileid, fileid_allowed, "add_months(obsdate, l2propint))" + \ " or(lower(" + accessid + ") in(select " + \ accessid + " from " + tmp_accessiddbtbl + ")))" - + elif (self.dbms.lower() == 'pgsql'): access_constraint = "((current_date > " + \ "(obsdate + (l2propint * '1 month'::interval)))" + \ " or (lower(" + accessid + ") in (select " + \ accessid + " from " + tmp_accessiddbtbl + ")))" - + else: if (self.dbms.lower() == 'oracle'): access_constraint = \ "(current_date > add_months(obsdate, l2propint))" - + elif (self.dbms.lower() == 'pgsql'): access_constraint = "(current_date > " + \ @@ -1916,7 +1914,7 @@ def __createTmpFileiddb__(self, tmp_fileiddbtbl, fileid, fileid_allowed, logging.debug(f'{self.msg:s}') raise Exception(self.msg) - + rowcnt_conditional = cursor.rowcount if self.debug: logging.debug('') @@ -2119,13 +2117,13 @@ def __dropDbtbl__(self, dbtable, **kwargs): logging.debug(f'table {dbtable:s} successfully dropped') except Exception as e: - + if self.debug: logging.debug('') logging.debug(f'drop table exception: {str(e):s}') - + pass - + return # diff --git a/TAP/tablenames.py b/TAP/tablenames.py index a3f230d..1ea9106 100644 --- a/TAP/tablenames.py +++ b/TAP/tablenames.py @@ -4,10 +4,10 @@ import itertools -import sqlparse -from sqlparse.sql import IdentifierList, Identifier -from sqlparse.tokens import Keyword, DML +import sqlparse +from sqlparse.sql import Identifier, IdentifierList +from sqlparse.tokens import DML, Keyword class TableNames: diff --git a/TAP/tap.py b/TAP/tap.py index c2102ae..ef6a4d8 100755 --- a/TAP/tap.py +++ b/TAP/tap.py @@ -2,34 +2,29 @@ # This code is released with a BSD 3-clause license. License information is at # https://github.com/Caltech-IPAC/nexsciTAP/blob/master/LICENSE -import os -import sys +import cgi +import datetime import fcntl import html - import logging - -import datetime -import time +import math +import os import signal -import math - -import cgi +import sys import tempfile +import time import xmltodict -from bs4 import BeautifulSoup - from ADQL.adql import ADQL - +from bs4 import BeautifulSoup from spatial_index import SpatialIndex -from TAP.tapquery import tapQuery from TAP.configparam import configParam from TAP.propfilter import propFilter from TAP.tablenames import TableNames -from TAP.vositables import vosiTables from TAP.tablevalidator import TableValidationError, TableValidator +from TAP.tapquery import tapQuery +from TAP.vositables import vosiTables class Tap: @@ -230,7 +225,7 @@ def __run__(self, **kwargs): self.uwsheader = '' - + for key in self.form: if self.debug: logging.debug(f' key: {key:<15} val: {self.form[key].value:s}') @@ -244,7 +239,7 @@ def __run__(self, **kwargs): if(key.lower() == 'lang'): self.lang = self.form[key].value self.param['lang'] = self.form[key].value - + if(key.lower() == 'request'): self.param['request'] = self.form[key].value @@ -305,7 +300,7 @@ def __run__(self, **kwargs): self.maxrec = int(maxrec_dbl) - except Exception as e: + except Exception: self.msg = "Failed to convert input maxrec value [" + \ self.param['maxrec'] + "] to integer." @@ -314,7 +309,7 @@ def __run__(self, **kwargs): if (len(self.lang) == 0): self.lang = 'ADQL' - self.param['lang'] = 'ADQL' + self.param['lang'] = 'ADQL' self.nparam = len(self.param) @@ -354,11 +349,11 @@ def __run__(self, **kwargs): self.configext = arr[0] arr = arr[1:] narr = len(arr) - + if self.debug: logging.debug('') logging.debug(f'id= {self.id:s} configext= {self.configext:s}') - + if(arr[0] == 'async'): self.tapcontext = 'async' @@ -370,7 +365,7 @@ def __run__(self, **kwargs): self.tapcontext = 'capabilities' elif (arr[0] == 'tables'): self.tapcontext = 'tables' - + if (len(self.tapcontext) == 0): self.msg = 'PATH_INFO: sync or async not found.' self.__printError__('votable', self.msg, errcode='404') @@ -382,17 +377,17 @@ def __run__(self, **kwargs): if(narr > 1 and len(arr[1]) > 0): # - #{ narr > 1: + #{ narr > 1: # input query contains more than TAP/(sync/async)/statuskey for - # retrive status.xml, we set getstatus=1; - # + # retrive status.xml, we set getstatus=1; + # self.getstatus = 1 self.id = arr[1] if self.debug: logging.debug('') logging.debug(f'id= {self.id:s} getstatus= {self.getstatus:d}') - + if (len(self.id) == 0): self.msg = 'Failed to find jobid for getStatus request.' self.__printError__('votable', self.msg, errcode='404') @@ -400,7 +395,7 @@ def __run__(self, **kwargs): len_id = len(self.id) ind = self.pathinfo.find(self.id) - + i = ind + len_id + 1 self.statuskey = self.pathinfo[i:] @@ -423,10 +418,10 @@ def __run__(self, **kwargs): logging.debug(f'setstatus = {self.setstatus:d}') logging.debug(f'id = {self.id:s}') # - #} end parsing pathinfo for narr > 1: + #} end parsing pathinfo for narr > 1: # input query contains more than TAP/(sync/async)/statuskey for - # retrive status.xml, we set getstatus=1; - # + # retrive status.xml, we set getstatus=1; + # self.cookiestr = os.getenv('HTTP_COOKIE', default='') @@ -436,7 +431,7 @@ def __run__(self, **kwargs): logging.debug(f'cookiestr (http) = {self.cookiestr:s}') if (len(self.cookiestr) == 0): - + self.cookiestr = self.token if self.debug: logging.debug('') @@ -475,10 +470,10 @@ def __run__(self, **kwargs): self.config = None try: self.config = configParam(self.configpath, instance=self.instance, debug=self.debug) - + if self.debug: logging.debug('') - logging.debug(f'returned configParam:') + logging.debug('returned configParam:') logging.debug('%s', self.config) except Exception as e: @@ -486,7 +481,7 @@ def __run__(self, **kwargs): if self.debug: logging.debug('') logging.debug(f'config exception: {str(e):s}') - + self.msg = 'Internal system error: config variables read error.' self.__printError__('votable', str(e), errcode='500') @@ -500,7 +495,7 @@ def __run__(self, **kwargs): logging.debug('') logging.debug(f'workdir = {self.workdir:s}') - + self.arraysize = self.config.arraysize self.cookiename = self.config.cookiename @@ -528,7 +523,7 @@ def __run__(self, **kwargs): - # Special case: HTTP DELETE. We only need + # Special case: HTTP DELETE. We only need # the REQUEST_METHOD and PATH_INFO environment # variable. The first must be DELETE and the # second is the directory of th TAP job we wish @@ -548,7 +543,7 @@ def __run__(self, **kwargs): # logging.debug('') # logging.debug(f'DELETE: {delete_dir:s}') - + # # Initialize statdict dict # @@ -570,7 +565,7 @@ def __run__(self, **kwargs): self.statdict['destruction'] = '' self.statdict['endtime'] = '' self.statdict['duration'] = 0 - + self.statdict['stime'] = None self.statdict['resulturl'] = '' @@ -588,7 +583,7 @@ def __run__(self, **kwargs): # # { Make workspace: - # make TAP subdir if it doesn't exist, + # make TAP subdir if it doesn't exist, # make a workspace name with unique id # @@ -639,7 +634,7 @@ def __run__(self, **kwargs): else: # - # { input jobid exists: + # { input jobid exists: # Retrieve workspace from id # @@ -667,14 +662,14 @@ def __run__(self, **kwargs): # #{ if tapcontext is one of vosiEnpoint, take care of take care of VOSI - # output and return + # output and return # if (self.tapcontext == 'availability'): self.__printVosiAvailability__ () if (self.tapcontext == 'capabilities'): self.__printVosiCapability__ () - + # # vositable: make up vositbl filepath # @@ -686,11 +681,11 @@ def __run__(self, **kwargs): logging.debug(f'vosipath = {self.vosipath:s}') if (self.tapcontext == 'tables'): - + if self.debug: - + logging.debug('') - logging.debug(f'call printVosiTables:') + logging.debug('call printVosiTables:') logging.debug(f'statuspath = {self.statuspath:s}') self.__printVosiTables__ (self.config.connectInfo, \ @@ -702,7 +697,7 @@ def __run__(self, **kwargs): # #} end vosiEnpoint outputs # - + # # Make up status file name # @@ -720,7 +715,7 @@ def __run__(self, **kwargs): # # If async and phase == PENDING: send 303 with statusurl and exit # - # If async and phase != RUN and != ABORT, i.e. bogus value: + # If async and phase != RUN and != ABORT, i.e. bogus value: # return with 400 client error immediately. # @@ -740,13 +735,13 @@ def __run__(self, **kwargs): if ((self.tapcontext == 'async') and \ (self.getstatus == 0)): - + if (len(self.param['phase']) == 0): # # { If phase not specified: set to PENDING and exit # - + if self.debug: logging.debug('') logging.debug('case: set to PENDING') @@ -771,23 +766,23 @@ def __run__(self, **kwargs): # # } end of PENDING case # - + elif (self.param['phase'].lower() == 'abort'): # # { If phase is abort # - + if self.debug: logging.debug('') logging.debug('case bad phase input: abort') self.msg = "There is no existing job to be aborted." - + self.statdict['phase'] = 'ABORTED' self.statdict['jobid'] = self.workspace self.statdict['errmsg'] = self.msg - + """ self.__printError__('votable', self.msg, errcode='400') @@ -811,14 +806,14 @@ def __run__(self, **kwargs): logging.debug('Return HTTP redirect status.xml and exit.') sys.exit() - + elif ((self.param['phase'].lower() != 'run') and \ (self.param['phase'].lower() != 'abort')): - + # # { If phase is bogus value # - + if self.debug: logging.debug('') logging.debug('case bad phase input:') @@ -828,7 +823,7 @@ def __run__(self, **kwargs): self.statdict['jobid'] = self.workspace self.statdict['errmsg'] = self.msg - + """ self.__printError__('votable', self.msg, errcode='400') @@ -838,7 +833,7 @@ def __run__(self, **kwargs): sys.exit() """ - + self.__writeStatusMsg__(self.statuspath, self.statdict, self.param) @@ -858,41 +853,41 @@ def __run__(self, **kwargs): # # } end of async and getstatus=0 case # - - + + if ((self.tapcontext == 'async') and (self.getstatus == 1)): # #{ async and getstatus=1 case: # - + if (len(self.param['phase']) == 0): # - #{ async getStatus case: tap query includes jobid but - # input phase is blank + #{ async getStatus case: tap query includes jobid but + # input phase is blank # if self.debug: logging.debug('') logging.debug('case: getStatus') - + try: self.__getStatus__(self.workdir, self.id, self.statuskey, \ self.param) except Exception as e: - + self.__printError__(self.format, str(e)) - + # # } end getStatus and printError will exit when done # - + else: - # - # {Tap input with jobid and phase: need to + # + # {Tap input with jobid and phase: need to # parse statuspath to retrieve input parameters # if self.debug: logging.debug('') - logging.debug (f'case: retrieve parameters from workspace') + logging.debug ('case: retrieve parameters from workspace') logging.debug (f'statuspath= {self.statuspath:s}') xmlstruct = None @@ -968,10 +963,10 @@ def __run__(self, **kwargs): doc = xmltodict.parse (xmlstruct) job = doc['uws:job'] - + if self.debug: logging.debug ('') - logging.debug (f'job= ') + logging.debug ('job= ') logging.debug (job) self.statdict['jobid'] = job['uws:jobId'] @@ -986,24 +981,24 @@ def __run__(self, **kwargs): if self.debug: logging.debug ('') - logging.debug (f'param retrieved from status.xml:') - logging.debug (f'jobid: ') + logging.debug ('param retrieved from status.xml:') + logging.debug ('jobid: ') logging.debug (self.statdict["jobid"]) - logging.debug (f'process_id:') + logging.debug ('process_id:') logging.debug (self.statdict["process_id"]) - logging.debug (f'ownerId:') + logging.debug ('ownerId:') logging.debug (self.statdict["ownerId"]) - logging.debug (f'phase:') + logging.debug ('phase:') logging.debug (self.statdict["phase"]) - logging.debug (f'quote:') + logging.debug ('quote:') logging.debug (self.statdict["quote"]) - logging.debug (f'starttime:') + logging.debug ('starttime:') logging.debug (self.statdict["starttime"]) - logging.debug (f'endtime:') + logging.debug ('endtime:') logging.debug (self.statdict["endtime"]) - logging.debug (f'destruction:') + logging.debug ('destruction:') logging.debug (self.statdict["destruction"]) - logging.debug (f'duration:') + logging.debug ('duration:') logging.debug (self.statdict["duration"]) # @@ -1012,8 +1007,8 @@ def __run__(self, **kwargs): # #} end async and getstatus=1 case # - - + + # # Make result table names # @@ -1101,7 +1096,7 @@ def __run__(self, **kwargs): # self.__printAsyncResponse__(self.statusurl) - + if self.debug: logging.debug('') logging.debug(f'returned printAsyncResponse') @@ -1110,7 +1105,7 @@ def __run__(self, **kwargs): # # } end async RUN case # - + elif (self.param['phase'] == 'ABORT'): # # { async ABORT case @@ -1125,8 +1120,8 @@ def __run__(self, **kwargs): if self.debug: logging.debug('') - logging.debug(f'currently executing:') - + logging.debug('currently executing:') + pid = self.statdict['process_id'] if self.debug: logging.debug('') @@ -1135,8 +1130,8 @@ def __run__(self, **kwargs): try: os.kill (pid, signal.SIGKILL) - except Exception as e: - + except Exception: + if self.debug: logging.debug('') logging.debug(f'Error abort job: pid = {pid:d}') @@ -1152,97 +1147,97 @@ def __run__(self, **kwargs): destructtime = datetime.datetime.now() destruction = \ destructtime.strftime('%Y-%m-%dT%H:%M:%S.%f')[:-4] - + etime = datetime.datetime.now() endtime = etime.strftime('%Y-%m-%dT%H:%M:%S.%f')[:-4] if self.debug: logging.debug('') - logging.debug(f'got here0-3') - logging.debug(f'etime:') + logging.debug('got here0-3') + logging.debug('etime:') logging.debug(etime) - + starttimestr = self.statdict['starttime'] - + stime = datetime.datetime.strptime \ (starttimestr, '%Y-%m-%dT%H:%M:%S.%f') if self.debug: logging.debug('') - logging.debug(f'stime:') + logging.debug('stime:') logging.debug(stime) - - durationtime = etime - stime + + durationtime = etime - stime if self.debug: logging.debug('') - logging.debug(f'got here0-4') - logging.debug(f'durationtime:') + logging.debug('got here0-4') + logging.debug('durationtime:') logging.debug(durationtime) - + durationstr = str(durationtime.total_seconds())[:50] if self.debug: logging.debug('') - logging.debug(f'got here0-5') + logging.debug('got here0-5') logging.debug(f'durationstr= {durationstr:s}') - + duration_dbl = float(durationstr) if self.debug: logging.debug('') - logging.debug(f'got here0-6') - + logging.debug('got here0-6') + duration = math.ceil(duration_dbl) if self.debug: logging.debug('') - logging.debug(f'got here0-7') - + logging.debug('got here0-7') + self.statdict['endtime'] = endtime if self.debug: logging.debug('') - logging.debug(f'got here0-8') - + logging.debug('got here0-8') + self.statdict['duration'] = duration if self.debug: logging.debug('') - logging.debug(f'got here0-9') - + logging.debug('got here0-9') + self.statdict['destruction'] = destruction if self.debug: logging.debug('') - logging.debug(f'got here0-10') - + logging.debug('got here0-10') + self.statdict['phase'] = 'ABORTED' - self.statdict['errmsg'] = '' + self.statdict['errmsg'] = '' if self.debug: logging.debug('') - logging.debug(f'got here0-11') - + logging.debug('got here0-11') + """ self.__writeStatusMsg__(self.statuspath, self.statdict, self.param) self.__printAsyncResponse__(self.statusurl) - + if self.debug: logging.debug('') logging.debug(f'returned printAsyncResponse') """ - + elif (self.statdict['phase'] == 'PENDING'): errmsg = 'PENDING job aborted' if self.debug: logging.debug('') - logging.debug(f'call writeAsyncError:') + logging.debug('call writeAsyncError:') logging.debug(f'errmsg= {errmsg:s}') - - + + self.statdict['starttime'] = '' self.statdict['endtime'] = '' - self.statdict['duration'] = 0 - + self.statdict['duration'] = 0 + self.statdict['phase'] = 'ABORTED' self.statdict['errmsg'] = errmsg @@ -1251,7 +1246,7 @@ def __run__(self, **kwargs): self.param) self.__printAsyncResponse__(self.statusurl) - + if self.debug: logging.debug('') logging.debug(f'returned printAsyncResponse') @@ -1263,7 +1258,7 @@ def __run__(self, **kwargs): else: # - # { any other bogus values: this only occured in the initial + # { any other bogus values: this only occured in the initial # PENDING case; the other case that the client send in bogus # case first time around would have been returned by a HTTP # 400 error immediately. @@ -1273,14 +1268,14 @@ def __run__(self, **kwargs): logging.debug ('case bogus phase value') errmsg = "Unknown input job phase=" + self.param['phase'] - + self.statdict['starttime'] = '' self.statdict['endtime'] = '' - self.statdict['duration'] = 0 - + self.statdict['duration'] = 0 + self.statdict['phase'] = 'ERROR' self.statdict['errmsg'] = errmsg - + # # } end async bogus value case # @@ -1299,30 +1294,30 @@ def __run__(self, **kwargs): logging.debug ('call printAsyncResponse') self.__printAsyncResponse__(self.statusurl) - + if self.debug: logging.debug('') - logging.debug(f'returned printAsyncResponse') + logging.debug('returned printAsyncResponse') # - # } end async submit case + # } end async submit case # # - # { Before running query -- both sync and async cases, check - # input parameters to reject obvious input errors - # - + # { Before running query -- both sync and async cases, check + # input parameters to reject obvious input errors + # + if self.debug: logging.debug('') - logging.debug(f'continue: check lang') + logging.debug('continue: check lang') if (self.param['lang'].lower() != 'adql' and \ self.param['lang'].lower() != 'adql-2.0'): - + self.msg = "Input parameter (lang=" + self.param['lang'] + \ - ") error: only lang=ADQL is implemented." - + ") error: only lang=ADQL is implemented." + if(self.tapcontext == 'async'): self.__writeAsyncError__(self.msg, self.statuspath, self.statdict, self.param) @@ -1333,10 +1328,10 @@ def __run__(self, **kwargs): if ((self.tapcontext == 'async') and \ (self.param['phase'].lower() != 'run') and\ (self.param['phase'].lower() != 'abort')): - + self.statdict['phase'] = 'ERROR' self.statdict['jobid'] = self.workspace - + self.msg = "Unknown input job phase=" + self.param['phase'] self.__writeAsyncError__(self.msg, self.statuspath, @@ -1346,7 +1341,7 @@ def __run__(self, **kwargs): if self.debug: logging.debug('') - logging.debug(f'continue: check format') + logging.debug('continue: check format') if ((self.format != 'votable') and \ (self.format != 'ipac') and \ @@ -1372,7 +1367,7 @@ def __run__(self, **kwargs): if self.debug: logging.debug('') - logging.debug(f'continue: check maxrec') + logging.debug('continue: check maxrec') logging.debug(f'self.maxrecstr= {self.maxrecstr:s}') @@ -1541,7 +1536,7 @@ def __run__(self, **kwargs): if self.debug: logging.debug('') - logging.debug(f'ADQL initialized') + logging.debug('ADQL initialized') self.query = adql.sql(query_adql) @@ -1575,7 +1570,7 @@ def __run__(self, **kwargs): tables = tn.extract_tables(self.query) self.dbtable = tables[0] - except Exception as e: + except Exception: if self.debug: logging.debug('') logging.debug('TableName exception') @@ -1584,9 +1579,9 @@ def __run__(self, **kwargs): if len(self.dbtable) == 0: self.msg = 'No table name found in ADQL query.' - + if(self.tapcontext == 'async'): - + self.__writeAsyncError__(self.msg, self.statuspath, self.statdict, self.param) else: @@ -1694,7 +1689,7 @@ def __run__(self, **kwargs): if self.debug: logging.debug('') - logging.debug(f'tapQuery parameters:') + logging.debug('tapQuery parameters:') logging.debug(f'query = [{self.query:s}]') logging.debug(f'workdir = [{self.userWorkdir:s}]') logging.debug(f'format = [{self.format:s}]') @@ -1830,7 +1825,7 @@ def __run__(self, **kwargs): etime = datetime.datetime.now() durationtime = etime - self.statdict['stime'] - + if self.debug: logging.debug('') logging.debug('durationtime:') @@ -1845,7 +1840,7 @@ def __run__(self, **kwargs): if self.debug: logging.debug('') logging.debug(f'duration_dbl= {duration_dbl:f}') - + duration = math.ceil(duration_dbl) if self.debug: @@ -2020,7 +2015,7 @@ def __printStatus__(self, key, retval, outtype, **kwargs): # # Header # - + retvalstr = str(retval) if self.debug: logging.debug('') @@ -2034,16 +2029,16 @@ def __printStatus__(self, key, retval, outtype, **kwargs): uwsschema = ' xmlns:uws="http://www.ivoa.net/xml/UWS/v1.0" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:xsi="http://www.w3.org/2001/XMLSchema" xsi:schemaLocation="http://www.ivoa.net/xml/UWS/v1.0 http://www.ivoa.net/xml/UWS/v1.0">' - + if ((key == 'error') or (key == 'errorSummary') or (key == 'errmsg')): - + if(outtype == 'xml'): - self.__printError__ ('votable', retval) + self.__printError__ ('votable', retval) else: - self.__printError__ ('plain', retval) - + self.__printError__ ('plain', retval) + # # parameters and results vosi endpoints # @@ -2054,40 +2049,40 @@ def __printStatus__(self, key, retval, outtype, **kwargs): print("Content-type: text/xml\r") print("\r") print('') - + if(key == 'parameters'): - + if self.debug: logging.debug('') logging.debug('key = parameters') - + ind = retvalstr.index('>') if self.debug: logging.debug('') logging.debug('ind=') logging.debug(ind) - + if self.debug: logging.debug('') logging.debug(f'retvalstr[0:ind]= {retvalstr[0:ind-1]:s}') logging.debug(f'retvalstr[ind+1]= {retvalstr[ind+1]:s}') - - val = retvalstr[0:ind] + uwsschema + retvalstr[ind+1:] + + val = retvalstr[0:ind] + uwsschema + retvalstr[ind+1:] if self.debug: logging.debug('') logging.debug(f'val= {val:s}') - + print(val) sys.stdout.flush() elif(key == 'results'): - + ind = retvalstr.index('>') - val = retvalstr[0:ind] + uwsschema + retvalstr[ind+1:] + val = retvalstr[0:ind] + uwsschema + retvalstr[ind+1:] if self.debug: logging.debug('') logging.debug(f'val= {val:s}') - + print(val) sys.stdout.flush() @@ -2097,10 +2092,10 @@ def __printStatus__(self, key, retval, outtype, **kwargs): print (retval) sys.stdout.flush() - + if self.debug: logging.debug('Write status to user and exit.') - + sys.exit() # # } end of printStatus @@ -2112,7 +2107,7 @@ def __getStatus__(self, workdir, workspace, key, param, **kwargs): # # { initialize format to votable but after extracting format - # the output should follow the format + # the output should follow the format # if self.debug: logging.debug('') @@ -2120,7 +2115,7 @@ def __getStatus__(self, workdir, workspace, key, param, **kwargs): logging.debug('key= ') logging.debug(key) - + isExist = os.path.exists(self.statuspath) if(isExist == 0): @@ -2219,13 +2214,13 @@ def __getStatus__(self, workdir, workspace, key, param, **kwargs): logging.debug('') logging.debug('key == parameters') logging.debug('call __printStatus') - + self.__printStatus__('parameters', parameters, 'xml') # # } end key=parameters # - + elif ((key == 'phase') or (key == 'startTime') or (key == 'endTime') @@ -2246,7 +2241,7 @@ def __getStatus__(self, workdir, workspace, key, param, **kwargs): if self.debug: logging.debug('') logging.debug(\ - f'key == phase, executionduration or destruction etc.') + 'key == phase, executionduration or destruction etc.') # # { Single value return # @@ -2296,8 +2291,8 @@ def __getStatus__(self, workdir, workspace, key, param, **kwargs): if self.debug: logging.debug('') - logging.debug(f'key == error') - + logging.debug('key == error') + retval = '' errmsg = '' @@ -2305,11 +2300,11 @@ def __getStatus__(self, workdir, workspace, key, param, **kwargs): if self.debug: logging.debug('') - logging.debug(f'key == error') + logging.debug('key == error') try: errmsg = job['uws:errorSummary']['uws:message'] - except Exception as e: + except Exception: self.printError (\ self.format, 'error extracting errorSummary') @@ -2318,17 +2313,17 @@ def __getStatus__(self, workdir, workspace, key, param, **kwargs): logging.debug(f'errmsg: {errmsg:s}') if (self.format == 'votable'): - + if (len(errmsg) > 0): outstr = f' {errmsg:s}' else: - outstr = f' ' + outstr = ' ' self.__printStatus__('errorSummary', outstr, 'xml') - + else: if (len(errmsg) > 0): - outstr = errmsg + outstr = errmsg else: outstr = 'No error message' @@ -2351,7 +2346,7 @@ def __getStatus__(self, workdir, workspace, key, param, **kwargs): logging.debug(\ 'case results or results/result or results/resulturl') - + if (key == 'results'): # # { if results @@ -2359,7 +2354,7 @@ def __getStatus__(self, workdir, workspace, key, param, **kwargs): if self.debug: logging.debug('') logging.debug('key= results') - + if (phase.lower() == 'completed'): # # { if reulsts case and phase = completed @@ -2378,35 +2373,35 @@ def __getStatus__(self, workdir, workspace, key, param, **kwargs): # logging.debug(result) #results = job['uws:results'] - + #if self.debug: # logging.debug('') # logging.debug(f'results: {results:s}') - + #outstr = results - except Exception as e: - + except Exception: + if self.debug: logging.debug('') logging.debug('error retrieving results') - + self.__printError__(self.format, \ 'Failed to extract results', errcode='400') - + if self.debug: logging.debug('') logging.debug('results:') logging.debug(results) logging.debug('result:') logging.debug(result) - + self.__printStatus__(key, results, 'xml') sys.exit() - + + # + # } end results case # - # } end results case - # else: # # { if reulsts case but phase NOT completed @@ -2419,13 +2414,13 @@ def __getStatus__(self, workdir, workspace, key, param, **kwargs): # # } end if results # - else: + else: # # { if results/resulturl or 'results/result: # if (phase.lower() == 'completed'): # - # { if reulsts/resulturl or results/result cases + # { if reulsts/resulturl or results/result cases # and phase = completed # try: @@ -2433,12 +2428,12 @@ def __getStatus__(self, workdir, workspace, key, param, **kwargs): resulturl = \ job['uws:results']['uws:result']['@xlink:href'] - except Exception as e: - + except Exception: + if self.debug: logging.debug('') logging.debug('error retrieving result') - + self.__printError__ (self.format, \ 'Error retrieving result or resulturl') @@ -2452,7 +2447,7 @@ def __getStatus__(self, workdir, workspace, key, param, **kwargs): self.__printStatus__(key, resulturl, 'plain') sys.exit() - + elif (key == 'results/result'): indx = resulturl.find(workspace) @@ -2466,7 +2461,7 @@ def __getStatus__(self, workdir, workspace, key, param, **kwargs): fp = None try: fp = open(resultpath, 'r') - except Exception as e: + except Exception: msg = 'Failed to open result file: ' + resultpath self.__printError__(format, msg, errcode='400') @@ -2496,31 +2491,31 @@ def __getStatus__(self, workdir, workspace, key, param, **kwargs): fp.close() sys.exit() - + # - # } end results/resulturl or results/result cases + # } end results/resulturl or results/result cases # and phase completed - # - + # + else: # - # { if reulsts/resulturl or results/result cases + # { if reulsts/resulturl or results/result cases # and phase NOT completed # self.__printError__ (self.format, \ 'No result or resulturl because phase is not COMPLETED') - + # - # } end results/resulturl or results/result cases + # } end results/resulturl or results/result cases # and phase NOT completed - # - + # + # # } if results/resulturl or 'results/result: # # - # } end results or results/resulturl or results/result cases - # + # } end results or results/resulturl or results/result cases + # else: msg = f'key {key:s} is not a valid key' @@ -2535,8 +2530,8 @@ def __printError__(self, fmt, errmsg, **kwargs): # # { # - - errcode = '200' + + errcode = '200' if ('errcode' in kwargs): errcode = kwargs['errcode'] @@ -2562,7 +2557,7 @@ def __printError__(self, fmt, errmsg, **kwargs): if(self.infomsg[-1] == '?'): print(self.infomsg + 'dbtable=' + html.escape(self.dbtable)) else: - print(self.infomsg) + print(self.infomsg) print('') print('') @@ -2577,8 +2572,8 @@ def __printError__(self, fmt, errmsg, **kwargs): if(self.infomsg[-1] == '?'): print(self.infomsg + 'dbtable=' + self.dbtable) else: - print(self.infomsg) - + print(self.infomsg) + sys.stdout.flush() sys.exit() @@ -2603,22 +2598,22 @@ def __writeAsyncError__(self, errmsg, statuspath, statdict, param, **kwargs): # # Set status parameters # - + if self.debug: logging.debug('') - logging.debug(f'From writeAsyncError') - logging.debug(f'statdict["stime"]:') + logging.debug('From writeAsyncError') + logging.debug('statdict["stime"]:') logging.debug(statdict['stime']) - + if (statdict['stime'] is not None): - + etime = datetime.datetime.now() endtime = etime.strftime('%Y-%m-%dT%H:%M:%S.%f')[:-4] durationtime = etime - statdict['stime'] durationstr = str(durationtime.total_seconds())[:10] duration_dbl = float(durationstr) - + duration = math.ceil(duration_dbl) statdict['endtime'] = endtime @@ -2627,8 +2622,8 @@ def __writeAsyncError__(self, errmsg, statuspath, statdict, param, **kwargs): else: statdict['starttime'] = '' statdict['endtime'] = '' - statdict['duration'] = 0 - + statdict['duration'] = 0 + statdict['phase'] = 'ERROR' statdict['errmsg'] = errmsg @@ -2639,8 +2634,8 @@ def __writeAsyncError__(self, errmsg, statuspath, statdict, param, **kwargs): # # } end of writeAsyncError # - - + + def __printSyncResult__(self, resultpath, format, **kwargs): @@ -2758,9 +2753,9 @@ def __writeStatusMsg__(self, statuspath, statdict, param, **kwargs): if self.debug: logging.debug('') - logging.debug(f'Enter writeStatusMsg') + logging.debug('Enter writeStatusMsg') logging.debug(f'phase= {statdict["phase"]:s}') - + # # { TAP status result always written in XML format # input maxrecstr is stored in self.maxrecstr @@ -2769,7 +2764,7 @@ def __writeStatusMsg__(self, statuspath, statdict, param, **kwargs): format = param['format'].lower() maxrecstr = '' - + if (len(self.maxrecstr) == 0): maxrecstr = str(param['maxrec']) else: @@ -2779,13 +2774,13 @@ def __writeStatusMsg__(self, statuspath, statdict, param, **kwargs): logging.debug('') logging.debug(f'maxrecstr= {maxrecstr:s}') logging.debug(f"maxrec= {param['maxrec']:d}") - + fp = None try: fp = open(statuspath, 'w+') os.chmod(statuspath, 0o664) - except Exception as e: + except Exception: msg = 'Failed to open/create status file.' self.__printError__(format, msg) @@ -2815,7 +2810,7 @@ def __writeStatusMsg__(self, statuspath, statdict, param, **kwargs): errmsg = statdict['errmsg'] fp.write('\n') - + fp.write (self.uwsheader) fp.write ('\n') @@ -2861,7 +2856,7 @@ def __writeStatusMsg__(self, statuspath, statdict, param, **kwargs): if self.debug: logging.debug('') - logging.debug(f'wrote job status file') + logging.debug('wrote job status file') # @@ -2894,7 +2889,7 @@ def __writeStatusMsg__(self, statuspath, statdict, param, **kwargs): if self.debug: logging.debug('') - logging.debug(f'done writeAsyncError') + logging.debug('done writeAsyncError') # # Note: closing file automatically released the lock @@ -2911,7 +2906,7 @@ def __getDatalevel__(self, dbtable, **kwargs): # # { getDataLevel # - + if self.debug: logging.debug('') logging.debug('Enter __getDatalevel__') @@ -2950,12 +2945,12 @@ def __getDatalevel__(self, dbtable, **kwargs): # } end getDataLevel # - + def __printVosiTables__ (self, connectInfo, vosipath, **kwargs): # # { printVosiTables # - + if self.debug: logging.debug('') logging.debug('Enter __printVosiTables__') @@ -2979,13 +2974,13 @@ def __printVosiTables__ (self, connectInfo, vosipath, **kwargs): logging.debug('') logging.debug('call vosiTables') - vositbl = vosiTables (dbms=dbms, \ + vosiTables (dbms=dbms, \ dbserver=dbserver, \ userid=userid, \ password=password, \ outpath=vosipath, \ debug=1) - + if self.debug: logging.debug('') logging.debug('returned vosiTables') @@ -2997,7 +2992,7 @@ def __printVosiTables__ (self, connectInfo, vosipath, **kwargs): logging.debug('VosiTables exception:') logging.debug(f'exception: {str(e):s}') - self.__printError__ ('votable', str(e)) + self.__printError__ ('votable', str(e)) # # print out vosi table in the workspace @@ -3006,7 +3001,7 @@ def __printVosiTables__ (self, connectInfo, vosipath, **kwargs): fp = None try: fp = open(vosipath, 'r') - except Exception as e: + except Exception: msg = 'Failed to open vositable path: ' + vosipath self.__printError__('votable', msg, errcode='400') @@ -3037,7 +3032,7 @@ def __printVosiTables__ (self, connectInfo, vosipath, **kwargs): def __printVosiAvailability__ (self, **kwargs): - + # # { printVosiAvailability # @@ -3045,8 +3040,8 @@ def __printVosiAvailability__ (self, **kwargs): print ('') print ('') print ('') print (' true') print (' TAP service available.') @@ -3085,7 +3080,7 @@ def __printVosiCapability__ (self, **kwargs): print (' ') val = ' ' + self.httpurl + '/TAP' print (val) - + print (' ') print ('') print (' ') @@ -3137,10 +3132,10 @@ def __printVosiCapability__ (self, **kwargs): print (' ') print (' ') print (' ') - + val = ' ' + self.httpurl + '/TAP/tables' print (val) - + print (' GET') print (' application/xml') print (' ') @@ -3149,31 +3144,31 @@ def __printVosiCapability__ (self, **kwargs): print (' ') print (' ') print (' ') - + val = ' ' + self.httpurl + '/TAP/capabilities' print (val) - + print (' ') print (' ') print ('') print (' ') print (' ') print (' ') - + val = ' ' + self.httpurl + '/TAP/availability' print (val) - + print (' ') print (' ') print ('') - print ('') - + print ('') + sys.exit() - + # # } end printVosiCapability # - + # # } end tap class # diff --git a/TAP/tapquery.py b/TAP/tapquery.py index cf543ea..47bf0db 100644 --- a/TAP/tapquery.py +++ b/TAP/tapquery.py @@ -2,25 +2,20 @@ # This code is released with a BSD 3-clause license. License information is at # https://github.com/Caltech-IPAC/nexsciTAP/blob/master/LICENSE -import sys -import os -import logging - -import datetime - import argparse -import configobj - -from TAP.datadictionary import dataDictionary -from TAP.writeresult import writeResult -from TAP.tablenames import TableNames -from TAP.tablevalidator import TableValidator, TableValidationError -from TAP.configparam import configParam +import logging +import os +import sys from ADQL.adql import ADQL - from spatial_index import SpatialIndex +from TAP.configparam import configParam +from TAP.datadictionary import dataDictionary +from TAP.tablenames import TableNames +from TAP.tablevalidator import TableValidationError, TableValidator +from TAP.writeresult import writeResult + class tapQuery: @@ -80,8 +75,8 @@ def __init__(self, **kwargs): conn: We will need the above connectInfo for it's formatting info but we may have already made the DBMS connection elsewhere (for more involved - processing scenarios). In that case we can - optionally pass in the connection itself + processing scenarios). In that case we can + optionally pass in the connection itself instead of creating it here. query(char): the sql query to be executed, workdir(char): user work directory @@ -101,8 +96,8 @@ def __init__(self, **kwargs): workdir=userworkdir, filename=filename, maxrec=maxrec, - ddtbl=ddtbl, - ddtbl=ddfile, + ddtbl=ddtbl, + ddtbl=ddfile, format=format, racol=racol, deccol=deccol) @@ -120,7 +115,7 @@ def __init__(self, **kwargs): self.ddfile = kwargs['ddfile'] if self.debug: - logging.debug(f'Enter tapQuery') + logging.debug('Enter tapQuery') logging.debug(f'self.debug = {self.debug:d}') @@ -149,15 +144,15 @@ def __init__(self, **kwargs): # Otherwise collect the info for making the connection and make it. else: - + # Collect DBMS-specific keyword parameters # # There are two use modes for getting these parameters. Most of the time we let this code do everything; # getting the connection paramters and making the actual connection ('conn' parameter) to whichever DBMS # we are using. But sometimes we have a more complicated scenario (like using and populating temporary - # tables as part of the processing). The we will have run tapUtil() to do this same setup and used the + # tables as part of the processing). The we will have run tapUtil() to do this same setup and used the # database connection for these other steps before we get here. The all of the code here and the section - # below where we connect to the database will have already been executed there. + # below where we connect to the database will have already been executed there. # ORACLE @@ -166,7 +161,7 @@ def __init__(self, **kwargs): import cx_Oracle - self.dbserver = None + self.dbserver = None if('dbserver' in self.connectInfo): self.dbserver = self.connectInfo['dbserver'] @@ -176,7 +171,7 @@ def __init__(self, **kwargs): self.status = 'error' raise Exception(self.msg) - self.userid = None + self.userid = None if ('userid' in self.connectInfo): self.userid = self.connectInfo['userid'] @@ -208,7 +203,7 @@ def __init__(self, **kwargs): import sqlite3 - self.db = None + self.db = None if ('db' in self.connectInfo): self.db = self.connectInfo['db'] @@ -218,7 +213,7 @@ def __init__(self, **kwargs): self.status = 'error' raise Exception(self.msg) - self.tap_schema = None + self.tap_schema = None if('tap_schema' in self.connectInfo): self.tap_schema = self.connectInfo['tap_schema'] @@ -232,20 +227,20 @@ def __init__(self, **kwargs): logging.debug(f'db= {self.db:s}') logging.debug(f'tap_schema= {self.tap_schema:s}') - + # MYSQL elif self.dbms.lower() == 'mysql': import mysql.connector - - self.dbserver = None + + self.dbserver = None self.port = 3306 self.socket = None self.db = None self.userid = None self.password = None - + if ('dbserver' in self.connectInfo): self.dbserver = self.connectInfo['dbserver'] @@ -260,10 +255,10 @@ def __init__(self, **kwargs): if ((self.dbserver is None) and \ (self.socket is None)): - + self.msg = 'Failed to retrieve required input DB server ' \ 'parameter [dbserver] or [socket]' - + self.status = 'error' raise Exception(self.msg) @@ -306,7 +301,7 @@ def __init__(self, **kwargs): import psycopg2 - self.hostname = None + self.hostname = None if('hostname' in self.connectInfo): self.hostname = self.connectInfo['hostname'] @@ -316,7 +311,7 @@ def __init__(self, **kwargs): self.status = 'error' raise Exception(self.msg) - self.database = None + self.database = None if('database' in self.connectInfo): self.database = self.connectInfo['database'] @@ -326,7 +321,7 @@ def __init__(self, **kwargs): self.status = 'error' raise Exception(self.msg) - self.username = None + self.username = None if('username' in self.connectInfo): self.username = self.connectInfo['username'] @@ -336,7 +331,7 @@ def __init__(self, **kwargs): self.status = 'error' raise Exception(self.msg) - self.password = None + self.password = None if('password' in self.connectInfo): self.password = self.connectInfo['password'] @@ -379,14 +374,14 @@ def __init__(self, **kwargs): if self.debug: logging.debug('connected to Oracle, DB ' + self.dbserver) - except Exception as e: + except Exception: self.status = 'error' self.msg = 'Failed to connect to cx_Oracle' raise Exception(self.msg) - + # SQLITE3 elif self.dbms.lower() == 'sqlite3': @@ -411,7 +406,7 @@ def __init__(self, **kwargs): if self.debug: logging.debug('TAP_SCHEMA attached') - except Exception as e: + except Exception: self.status = 'error' self.msg = 'Failed to connect to SQLite3 databases' @@ -422,7 +417,7 @@ def __init__(self, **kwargs): # MYSQL elif self.dbms.lower() == 'mysql': - + try: if (self.dbserver is not None): @@ -433,7 +428,7 @@ def __init__(self, **kwargs): port=self.port, \ db=self.db ) - + elif (self.socket is not None): self.conn = mysql.connector.connect ( @@ -442,7 +437,7 @@ def __init__(self, **kwargs): unix_socket=self.socket, \ db=self.db ) - + else: self.status = 'error' self.msg = 'Failed to connect to mysql databases' @@ -451,7 +446,7 @@ def __init__(self, **kwargs): if self.debug: logging.debug('mysql connected') - except Exception as e: + except Exception: self.status = 'error' self.msg = 'Failed to connect to mysql databases' @@ -460,7 +455,7 @@ def __init__(self, **kwargs): if self.debug: logging.debug('here0') - + # POSTGRESQL @@ -485,10 +480,10 @@ def __init__(self, **kwargs): raise Exception(self.msg) - + # Get the query and query processing parameters (format, workspace, coordinate columns, etc.) - self.sql = None + self.sql = None if('query' in kwargs): self.sql = kwargs['query'] @@ -514,7 +509,7 @@ def __init__(self, **kwargs): if('filename' in kwargs): logging.debug(f'filename= {self.filename:s}') else: - logging.debug(f'filename= None') + logging.debug('filename= None') logging.debug(f'sql= {self.sql:s}') @@ -540,7 +535,7 @@ def __init__(self, **kwargs): try: self.maxrec = int(maxrecstr) - except Exception as e: + except Exception: self.msg = "Failed to convert input maxrec value [" + \ maxrecstr + "] to integer." @@ -553,7 +548,7 @@ def __init__(self, **kwargs): # Extract DB table name from query - self.dbtable = None + self.dbtable = None tn = TableNames() tables = tn.extract_tables(self.sql) @@ -626,12 +621,12 @@ def __init__(self, **kwargs): if self.debug: logging.debug('DD successfully retrieved.') - except Exception as e: + except Exception: if self.debug: logging.debug('dataDictionary retrieval failure.') - self.msg = f'dataDictionary retrieval failure.' + self.msg = 'dataDictionary retrieval failure.' raise Exception(self.msg) @@ -643,19 +638,19 @@ def __init__(self, **kwargs): # cursor = self.conn.cursor() - + if self.debug: logging.debug(f'sql = {self.sql:s}') logging.debug('call execut sql') self.msg = self.__executeSql__(cursor, self.sql, debug=1) - + if self.debug: logging.debug('returned executeSql') if len(self.msg) > 0: raise Exception(self.msg) - + # # Call writeResult @@ -703,10 +698,9 @@ def __executeSql__(self, cursor, sql, **kwargs): # { # - debug = 0 if('debug' in kwargs): - debug = kwargs['debug'] + kwargs['debug'] if self.debug: logging.debug('Enter executeSql') @@ -725,7 +719,7 @@ def __executeSql__(self, cursor, sql, **kwargs): try: cursor.execute(sql) - + except Exception as e: return_message = str(e).replace('"', "'") @@ -734,10 +728,10 @@ def __executeSql__(self, cursor, sql, **kwargs): try: self.conn.rollback() - - except Exception as e: + + except Exception: pass - + return return_message # diff --git a/TAP/taputil.py b/TAP/taputil.py index accc497..af4c19f 100644 --- a/TAP/taputil.py +++ b/TAP/taputil.py @@ -2,17 +2,15 @@ # This code is released with a BSD 3-clause license. License information is at # https://github.com/Caltech-IPAC/nexsciTAP/blob/master/LICENSE -import sys -import os import logging +import os +import sys -import pprint - -import configobj +from ADQL.adql import ADQL +from spatial_index import SpatialIndex from TAP.configparam import configParam -from ADQL.adql import ADQL -from spatial_index import SpatialIndex + class tapUtil: @@ -20,7 +18,7 @@ def __init__(self, **kwargs): self.conn = None - pid = os.getpid() + os.getpid() sys.tracebacklimit = 0 @@ -102,7 +100,7 @@ def __init__(self, **kwargs): socket = None if ('socket' in connectInfo): socket = connectInfo['socket'] - + port = None if ('port' in connectInfo): port = connectInfo['port'] @@ -174,7 +172,7 @@ def __init__(self, **kwargs): logging.debug('') logging.debug('connected to Oracle, DB ' + dbserver) - except Exception as e: + except Exception: msg = 'Failed to connect to cx_Oracle' raise Exception(msg) @@ -234,7 +232,7 @@ def __init__(self, **kwargs): logging.debug('') logging.debug('TAP_SCHEMA attached') - except Exception as e: + except Exception: msg = 'Failed to connect to SQLite3 databases' raise Exception(msg) @@ -242,9 +240,8 @@ def __init__(self, **kwargs): # MYSQL elif (dbms.lower() == 'mysql'): - - import msql.connector - + + try: if (dbserver is not None): @@ -255,7 +252,7 @@ def __init__(self, **kwargs): port=port, \ db=db ) - + elif (socket is not None): self.conn = mysql.connector.connect ( @@ -264,7 +261,7 @@ def __init__(self, **kwargs): unix_socket=socket, \ db=db ) - + else: msg = 'Failed to connect to mysql databases' raise Exception(msg) @@ -273,14 +270,14 @@ def __init__(self, **kwargs): logging.debug('') logging.debug('mysql connected') - except Exception as e: + except Exception: msg = 'Failed to connect to mysql databases' raise Exception(msg) if debug: logging.debug('') logging.debug('here0') - + else: msg = 'Invalid DBMS' raise Exception(msg) diff --git a/TAP/vositables.py b/TAP/vositables.py index 206bd69..58cba34 100644 --- a/TAP/vositables.py +++ b/TAP/vositables.py @@ -3,13 +3,10 @@ # https://github.com/Caltech-IPAC/nexsciTAP/blob/master/LICENSE -import sys -import os -import logging - -import datetime - import argparse +import logging +import os +import sys class vosiTables: @@ -27,7 +24,7 @@ class vosiTables: returnMsg = "" arraysize = 10000 - + # # DD columns # @@ -52,28 +49,28 @@ class vosiTables: coldesc = 0 nschema = 0 - + schema_namearr = [] schema_descarr = [] - - + + def __init__(self, **kwargs): - + # # { vosiTables.init method # """ - vosiTables class generates the VOSI compatible XML outputs for TAP - query with endpoint TAP/tables + vosiTables class generates the VOSI compatible XML outputs for TAP + query with endpoint TAP/tables Required keyword input parameters: - connectInfo (python dictionary): - a python dictionary containing info needed to make - a "connection" to a DBMS. - - ourpath(char): output table file path + connectInfo (python dictionary): + a python dictionary containing info needed to make + a "connection" to a DBMS. + + ourpath(char): output table file path Usage: @@ -86,13 +83,13 @@ def __init__(self, **kwargs): debug=debug) """ - self.dbms = '' + self.dbms = '' self.dbserver = '' self.userid = '' self.password = '' self.outpath = '' - self.debug = 0 - + self.debug = 0 + self.db = '' self.tap_schema = '' @@ -127,19 +124,19 @@ def __init__(self, **kwargs): # # Get keyword parameters # - + if ((self.dbms is None) or (len(self.dbms) == 0)): self.msg = 'Database name missing.' self.status = 'error' #self.__printVosiError__ (self.msg) - + raise Exception (self.msg) if self.debug: logging.debug('') logging.debug(f'dbms= {self.dbms:s}') - - + + if (self.dbms.lower() == 'oracle'): if ((self.dbserver is None) or (len(self.dbserver) == 0)): @@ -148,33 +145,33 @@ def __init__(self, **kwargs): #self.__printVosiError__ (self.msg) raise Exception (self.msg) - + if self.debug: logging.debug('') logging.debug(f'dbserver= {self.dbserver:s}') - + if ((self.userid is None) or (len(self.userid) == 0)): self.msg = 'Database userid missing.' self.status = 'error' #self.__printVosiError__ (self.msg) raise Exception (self.msg) - + if self.debug: logging.debug('') logging.debug(f'userid= {self.userid:s}') - + if ((self.password is None) or (len(self.password) == 0)): self.msg = 'Database password missing.' self.status = 'error' #self.__printVosiError__ (self.msg) raise Exception (self.msg) - + if self.debug: logging.debug('') logging.debug(f'password= {self.password:s}') - + elif (self.dbms.lower() == 'sqlite3'): if ((self.db is None) or (len(self.db) == 0)): @@ -183,30 +180,30 @@ def __init__(self, **kwargs): #self.__printVosiError__ (self.msg) raise Exception (self.msg) - + if self.debug: logging.debug('') logging.debug(f'db= {self.db:s}') - + if ((self.tap_schema is None) or (len(self.tap_schema) == 0)): - + self.msg = 'Database table name missing.' self.status = 'error' #self.__printVosiError__ (self.msg) raise Exception (self.msg) - + if self.debug: logging.debug('') logging.debug(f'tap_schema= {self.tap_schema:s}') - + if ((len(self.outpath) == 0) or (self.outpath is None)): self.msg = 'Required output path is missing.' #self.__printVosiError__ (self.msg) raise Exception (self.msg) - + if self.debug: logging.debug('') logging.debug(f'outpath= {self.outpath:s}') @@ -234,7 +231,7 @@ def __init__(self, **kwargs): logging.debug('') logging.debug('connected to Oracle, DB ' + self.dbserver) - except Exception as e: + except Exception: self.status = 'error' self.msg = 'Failed to connect to cx_Oracle' @@ -301,7 +298,7 @@ def __init__(self, **kwargs): logging.debug('') logging.debug('TAP_SCHEMA attached') - except Exception as e: + except Exception: self.status = 'error' self.msg = 'Failed to connect to SQLite3 databases' @@ -319,10 +316,10 @@ def __init__(self, **kwargs): # # } end Connect to DBMS # - + # # start loop for constructing output: - # + # # -- vosi header: tableset tag, # -- query for schemas: schema tag, # -- for each schema, query tables: table tag, @@ -333,25 +330,25 @@ def __init__(self, **kwargs): # -- end table tag, # -- end schema tag # -- end tableset tag - + # # open ouput file in workdir for vosi table output # #vositblpath = self.workdir + '/vositable.xml' vositblpath = self.outpath - + if self.debug: logging.debug('') logging.debug(f'vositblpath = {vositblpath:s}') - + fp = None try: fp = open (vositblpath, 'w') os.chmod(vositblpath, 0o664) except Exception as e: - + if self.debug: logging.debug('') logging.debug(f'open voditblpath exception: {str(e):s}') @@ -392,7 +389,7 @@ def __init__(self, **kwargs): if self.debug: logging.debug('') logging.debug(f'executeSql exception: {str(e):s}') - + #self.__printVosiError__ (str(e)) raise Exception (str(e)) @@ -409,7 +406,7 @@ def __init__(self, **kwargs): self.__getSchemaQueryArr__ (cursor, \ debug=self.debug) - + if self.debug: logging.debug('') logging.debug('returned getSchemaDesc') @@ -423,9 +420,9 @@ def __init__(self, **kwargs): #self.__printVosiError__ (str(e)) raise Exception (str(e)) - + self.nschema = len (self.schema_namearr) - + if self.debug: logging.debug('') logging.debug(f'nschema= {self.nschema:d}') @@ -436,7 +433,7 @@ def __init__(self, **kwargs): logging.debug(f'{i:d} schema_name={self.schema_namearr[i]:s}') logging.debug(f'schema_description={self.schema_descarr[i]:s}') - for ischema in range (0, self.nschema): + for ischema in range (0, self.nschema): # # { ischema loop # @@ -448,9 +445,9 @@ def __init__(self, **kwargs): self.schema_namearr[ischema] + "\n") fp.write(" " + \ self.schema_descarr[ischema] + "\n") - fp.flush() - - + fp.flush() + + # # 2. execute table sql # @@ -476,7 +473,7 @@ def __init__(self, **kwargs): if self.debug: logging.debug('') logging.debug(f'executeSql exception: {str(e):s}') - + #self.__printVosiError__ (str(e)) raise Exception (str(e)) @@ -487,15 +484,15 @@ def __init__(self, **kwargs): logging.debug('') logging.debug('call getTableQueryArr') - # - # 3. getSqlresult of table sql + # + # 3. getSqlresult of table sql # re-initialize arrays # table_namearr = [] table_descarr = [] table_typearr = [] - + ntable = 0 try: ntable = self.__getTableQueryArr__(cursor, table_namearr, \ @@ -506,7 +503,7 @@ def __init__(self, **kwargs): if self.debug: logging.debug('') logging.debug(f'getTableQueryArr exception: {str(e):s}') - + #self.__printVosiError__ (str(e)) raise Exception (str(e)) @@ -515,9 +512,9 @@ def __init__(self, **kwargs): logging.debug('') logging.debug('returned getTableQueryArr') logging.debug(f'ntable= {ntable:d}') - - - for itable in range (0, ntable): + + + for itable in range (0, ntable): logging.debug ('') logging.debug (f'itable= {itable:d}') logging.debug ( @@ -526,15 +523,15 @@ def __init__(self, **kwargs): f'table_desc= {table_descarr[itable]:s}') logging.debug ( f'table_type= {table_typearr[itable]:s}') - + # # for each table: # - for itable in range (0, ntable): + for itable in range (0, ntable): # # { itable loop - # + # # 1. write header block for each table, # fp.write(" \n") - fp.flush() - + fp.flush() + # # } end itable loop # - + # # 2. write end block for each schema # fp.write(" \n") - fp.flush() + fp.flush() # # } end ischema loop and write end block for xml file # fp.write("\n") - fp.flush() - + fp.flush() + # # } end vosiTables.init # @@ -702,7 +699,7 @@ def __getTableQueryArr__ (self, cursor, table_namearr, table_descarr, \ # table_name, description, and table_type; # debug = 0 - + if('debug' in kwargs): debug = kwargs['debug'] @@ -710,7 +707,6 @@ def __getTableQueryArr__ (self, cursor, table_namearr, table_descarr, \ logging.debug('') logging.debug('Enter getTableQueryArr') - size = 0 table_colname = [] table_dbtype = [] @@ -752,7 +748,6 @@ def __getTableQueryArr__ (self, cursor, table_namearr, table_descarr, \ dbtype = '' - size = None dbdatatypestr = str(col[1]) @@ -813,7 +808,7 @@ def __getTableQueryArr__ (self, cursor, table_namearr, table_descarr, \ logging.debug(f'nrec = {nrec:d}') logging.debug('') - + for ll in range(0, nrec): # @@ -821,7 +816,7 @@ def __getTableQueryArr__ (self, cursor, table_namearr, table_descarr, \ # row = rows[ll] - + if debug: logging.debug ('') logging.debug (f'len(row)= {len(row):d}') @@ -845,13 +840,13 @@ def __getTableQueryArr__ (self, cursor, table_namearr, table_descarr, \ if debug: logging.debug (f'i= {i:d}, data= {data:s}') logging.debug (f'table_colname= {table_colname[i]:s}') - + if (table_colname[i].lower() == 'table_name'): table_namearr.append (data) - + elif (table_colname[i].lower() == 'description'): table_descarr.append (data) - + elif (table_colname[i].lower() == 'table_type'): table_typearr.append (data) @@ -863,7 +858,7 @@ def __getTableQueryArr__ (self, cursor, table_namearr, table_descarr, \ # # } end of ll loop - # + # if debug: logging.debug('') logging.debug(table_namearr) @@ -871,7 +866,7 @@ def __getTableQueryArr__ (self, cursor, table_namearr, table_descarr, \ logging.debug(table_typearr) logging.debug('----------------------------------------') - + ntot = ntot + nrec if(len(rows) < cursor.arraysize): @@ -891,11 +886,11 @@ def __getTableQueryArr__ (self, cursor, table_namearr, table_descarr, \ # - + def __getSchemaQueryArr__ (self, cursor, **kwargs): # # { vosiTables.getSchemaQueryArr - # + # # this query contains only two columns: # schema_name and schema_description # @@ -906,12 +901,12 @@ def __getSchemaQueryArr__ (self, cursor, **kwargs): if self.debug: logging.debug('') - logging.debug(f'Enter __getSchemaQueryArr__\n') + logging.debug('Enter __getSchemaQueryArr__\n') size = 0 nfetch = self.arraysize cursor.arraysize = nfetch - + if debug: logging.debug('') logging.debug(f'nfetch= {nfetch:d}') @@ -938,7 +933,7 @@ def __getSchemaQueryArr__ (self, cursor, **kwargs): # colname = str(col[0]).lower() - + if debug: logging.debug('') logging.debug(f'colname(lower) = {colname:s}') @@ -951,8 +946,6 @@ def __getSchemaQueryArr__ (self, cursor, **kwargs): dbdatatype = '' size = None - precision = None - scale = None dbdatatypestr = str(col[1]) @@ -1079,7 +1072,7 @@ def __getSchemaQueryArr__ (self, cursor, **kwargs): # row = rows[ll] - + if debug: logging.debug(row) logging.debug(f'row[0]= {row[0]:s}') @@ -1088,7 +1081,7 @@ def __getSchemaQueryArr__ (self, cursor, **kwargs): self.schema_namearr.append(row[0]) self.schema_descarr.append(row[1]) - + if debug: logging.debug(row) logging.debug(self.schema_namearr) @@ -1167,7 +1160,6 @@ def __writeOneTableResult__ (self, cursor, fp, **kwargs): # dbdatatype = '' - size = None dbdatatypestr = str(col[1]) @@ -1191,7 +1183,7 @@ def __writeOneTableResult__ (self, cursor, fp, **kwargs): # # } end extract dbdatatype from col_array: col[1] # - + i = i + 1 # @@ -1234,13 +1226,13 @@ def __writeOneTableResult__ (self, cursor, fp, **kwargs): # row = rows[ll] - data = '' + data = '' fp.write(" \n") - + col_name = '' for i in range(0, len(row)): - + if (row[i] is None): data = 'None' elif (dbtype[i] == 'STRING'): @@ -1260,65 +1252,65 @@ def __writeOneTableResult__ (self, cursor, fp, **kwargs): if self.debug: logging.debug('') logging.debug(f'll= {ll:d} col_name= {col_name:s} data= {data:s}') - + if (colnamearr[i] == 'column_name'): fp.write(" " + data + "\n") - + elif (colnamearr[i] == 'description'): fp.write(" " + \ "\n") - + elif (colnamearr[i] == 'datatype'): fp.write (" " + \ "" + \ data + "\n") - + elif (colnamearr[i] == 'arraysize'): if (data != 'None'): fp.write (" " + data + \ "\n") - + elif (colnamearr[i] == 'xtype'): if (data != 'None'): fp.write (" " + data + \ "\n") - + elif (colnamearr[i] == 'utype'): if (data != 'None'): fp.write(" " + data + \ "\n") - + elif (colnamearr[i] == 'unit'): if (data != 'None'): fp.write(" " + data + \ "\n") - + elif (colnamearr[i] == 'ucd'): if (data != 'None'): fp.write (" " + data + \ "\n") - + elif (colnamearr[i] == 'indexed'): if (data != 'None'): fp.write (" " + data + \ "\n") - + elif (colnamearr[i] == 'principal'): if (data != 'None' and data != 0 and data != '0'): fp.write(" " + \ "" + data + "\n") - + elif (colnamearr[i] == 'std'): if (data != 'None'): fp.write (" " + data + \ "\n") - + elif (colnamearr[i] == 'column_index'): if (data != 'None' and data != 0 and data != '0'): fp.write(" " + "" + data + \ "\n") - + irow = irow + 1 #if col_datatype == 'timestamp': @@ -1327,7 +1319,7 @@ def __writeOneTableResult__ (self, cursor, fp, **kwargs): # col_datatype = 'char' fp.write(" \n") - + # # } end of ll loop @@ -1336,7 +1328,7 @@ def __writeOneTableResult__ (self, cursor, fp, **kwargs): ntot = ntot + nrec if(len(rows) < cursor.arraysize): - break + break if self.debug: logging.debug('') @@ -1354,12 +1346,12 @@ def __writeOneTableResult__ (self, cursor, fp, **kwargs): def __writeForeignKey__ (self, cursor, fp, **kwargs): - + # # {vosiTables.writeForeignKey # debug = 0 - + if('debug' in kwargs): debug = kwargs['debug'] @@ -1404,7 +1396,6 @@ def __writeForeignKey__ (self, cursor, fp, **kwargs): # dbdatatype = '' - size = None dbdatatypestr = str(col[1]) @@ -1427,7 +1418,7 @@ def __writeForeignKey__ (self, cursor, fp, **kwargs): # # } end extract dbdatatype from col_array: col[1] # - + i = i + 1 # @@ -1466,16 +1457,15 @@ def __writeForeignKey__ (self, cursor, fp, **kwargs): # row = rows[ll] - - keyid = '' + targettbl = '' fromcol = '' targetcol= '' desc = '' - data = '' + data = '' for i in range(0, len(row)): - + if (row[i] is None): data = 'None' elif (dbtype[i] == 'STRING'): @@ -1488,7 +1478,7 @@ def __writeForeignKey__ (self, cursor, fp, **kwargs): logging.debug(f'i= {i:d}, data= {data:s}') if (i == 0): - keyid = data + pass elif (i == 1): targettbl = data elif (i == 2): @@ -1499,14 +1489,14 @@ def __writeForeignKey__ (self, cursor, fp, **kwargs): desc = data irow = irow + 1 - + # # for row, write column block # fp.write(" \n") fp.write(" " + targettbl + \ "\n") - + fp.write(" \n") fp.write(" " + fromcol + \ "\n") @@ -1525,7 +1515,7 @@ def __writeForeignKey__ (self, cursor, fp, **kwargs): ntot = ntot + nrec if(len(rows) < cursor.arraysize): - break + break if debug: logging.debug('') @@ -1585,7 +1575,7 @@ def __printVosiError__(self, errmsg): httphdr = "HTTP/1.1 500 ERROR\r" #print("HTTP/1.1 200 OK\r") - + print(httphdr) print("Content-type: text/xml\r") @@ -1628,19 +1618,19 @@ def main(): parser.add_argument('--dbserver', required=False, help='Oracle database server (dbserver) to use.') - + parser.add_argument('--userid', required=False, help='Oracle dbserver userid (userid) to use.') - + parser.add_argument('--password', required=False, help='Oracle dbserver (password) to use.') - + parser.add_argument('--db', required=False, help='Sqlite database name to use.') - + parser.add_argument('--tap_schema', required=False, help='Sqlite database table name to use.') - + parser.add_argument('--outpath', required=True, help='outpath for the result.') @@ -1662,47 +1652,47 @@ def main(): if (args.dbms is not None): dbms = args.dbms print (f'dbms= {dbms:s}') - + if (args.dbserver is not None): dbserver = args.dbserver print (f'dbserver= {dbserver:s}') - + if (args.userid is not None): userid = args.userid print (f'userid= {userid:s}') - - + + if (args.password is not None): password = args.password print (f'password= {password:s}') - - + + if (args.outpath is not None): outpath = args.outpath print (f'outpath= {outpath:s}') - - + + if (args.debug is not None): debug = args.debug print (f'debug= {debug:s}') - + if (args.db is not None): db = args.db print (f'db= {db:s}') - + if (args.tap_schema is not None): tap_schema = args.tap_schema print (f'tap_schema= {tap_schema:s}') - - + + if debug: debugfname = './vosi.debug' logging.basicConfig(filename=debugfname, \ level=logging.DEBUG) - print (f'debug turned on') - + print ('debug turned on') + logging.debug('') logging.debug('debug turned on') logging.debug(f'dbms= {dbms:s}') @@ -1713,13 +1703,12 @@ def main(): logging.debug(f'debug= {debug:s}') - vositbl = None try: if debug: logging.debug('') logging.debug('call vosiTables') - - vositbl = vosiTables (dbms=dbms, \ + + vosiTables (dbms=dbms, \ dbserver=dbserver, \ userid=userid, \ password=password, \ diff --git a/TAP/writeresult.py b/TAP/writeresult.py index 3489556..a7a24e1 100644 --- a/TAP/writeresult.py +++ b/TAP/writeresult.py @@ -13,10 +13,9 @@ # import logging -import datetime - from TAP import writerecs + class writeResult: time00 = None @@ -70,7 +69,7 @@ def __init__(self, cursor, workdir, dd, **kwargs): """ cursor: oracle query returned cursor containing data stream, workdir: work directory for output files, - filename: output file name. + filename: output file name. dd: data dictionary for the output columns, coldes(0/1): indictes whether to include column descriptions in the output file header @@ -103,8 +102,8 @@ def __init__(self, cursor, workdir, dd, **kwargs): logging.debug('') logging.debug('Enter writeResult:') logging.debug(f'self.debug= {self.debug:d}') - - + + self.cursor = cursor self.workdir = workdir self.dd = dd @@ -116,7 +115,7 @@ def __init__(self, cursor, workdir, dd, **kwargs): if self.debug: logging.debug('') logging.debug(f'ncols_dd= {self.ncols_dd:d}') - + if('racol' in kwargs): self.racol = kwargs['racol'] @@ -125,7 +124,7 @@ def __init__(self, cursor, workdir, dd, **kwargs): if('deccol' in kwargs): self.deccol = kwargs['deccol'] - + if (self.dd != None): self.ind_deccol = self.__getDDIndex__(self.dd, self.deccol) @@ -134,12 +133,12 @@ def __init__(self, cursor, workdir, dd, **kwargs): if('dbms' in kwargs): self.dbms = kwargs['dbms'] - + if self.debug: logging.debug('') logging.debug(f'dbms= {self.dbms:s}') - - + + if('format' in kwargs): self.format = kwargs['format'] @@ -149,7 +148,7 @@ def __init__(self, cursor, workdir, dd, **kwargs): if self.debug: logging.debug('') logging.debug('here0-2') - + self.coldesc = 0 if('coldesc' in kwargs): self.coldesc = kwargs['coldesc'] @@ -172,7 +171,7 @@ def __init__(self, cursor, workdir, dd, **kwargs): logging.debug(f' arraysize = {self.arraysize:d}') self.status = '' - + # # open querypath for output # @@ -191,9 +190,9 @@ def __init__(self, cursor, workdir, dd, **kwargs): resulttbl = 'result.tsv' elif(self.format == 'json'): resulttbl = 'result.json' - + self.outpath = self.workdir + '/' + resulttbl - + if self.debug: pathstr = str(self.outpath) logging.debug('') @@ -220,14 +219,14 @@ def __init__(self, cursor, workdir, dd, **kwargs): logging.debug('------------------------------------------------') for desc in self.cursor.description: - name = desc[0] + desc[0] coltype = desc[1] - dispsize = desc[2] - internalsz = desc[3] + desc[2] + desc[3] precision = desc[4] - scale = desc[5] - nullok = desc[6] - + scale = desc[5] + desc[6] + # @@ -255,9 +254,9 @@ def __init__(self, cursor, workdir, dd, **kwargs): # varchr, sting -- char, # int, integer, long, etc.. -- int (32bit), # - # oracle: dbtype == NUMBER, check first batch of data to see if - # it is int or float: if any number in this batch is float, - # consider the column dtype as double + # oracle: dbtype == NUMBER, check first batch of data to see if + # it is int or float: if any number in this batch is float, + # consider the column dtype as double # # pgsql: dbtype == decimal, assume it is double # @@ -268,19 +267,19 @@ def __init__(self, cursor, workdir, dd, **kwargs): dbdatatype = None - precision = None + precision = None scale = None - size = None + size = None i = 0 for col in self.cursor.description: # # { cursor description for loop: - # + # # cursor description is a list with ncols tuples: # each tuple contains each output colname's name, datatype, - # dbdatatype, size, precision, scale but most DBMSs' - # implementation are incomplete, only name and dbdatatype + # dbdatatype, size, precision, scale but most DBMSs' + # implementation are incomplete, only name and dbdatatype # are reliably implemented. # @@ -317,14 +316,14 @@ def __init__(self, cursor, workdir, dd, **kwargs): if self.debug: logging.debug('') - logging.debug(f'analyze description array:') + logging.debug('analyze description array:') logging.debug(f'dbms= {self.dbms:s}') if (self.dbms.lower() == 'oracle'): # # { oracle datatype from descriptor - # + # if self.debug: logging.debug('') logging.debug('oracle:') @@ -341,7 +340,7 @@ def __init__(self, cursor, workdir, dd, **kwargs): ind = dbdatatypestr.find("VARCHAR") if(ind != -1): dbdatatype = 'VARCHAR' - + ind = dbdatatypestr.find("STRING") if(ind != -1): dbdatatype = 'STRING' @@ -369,7 +368,7 @@ def __init__(self, cursor, workdir, dd, **kwargs): ind = dbdatatypestr.find("FLOAT") if(ind != -1): dbdatatype = 'FLOAT' - + if self.debug: logging.debug('') logging.debug(f'dbdatatype = {dbdatatype:s}') @@ -381,7 +380,7 @@ def __init__(self, cursor, workdir, dd, **kwargs): elif (self.dbms.lower() == 'pgsql'): # # { pgsql datatype from descriptor - # + # if self.debug: logging.debug('') logging.debug('pgsql:') @@ -439,18 +438,18 @@ def __init__(self, cursor, workdir, dd, **kwargs): elif (self.dbms.lower() == 'mysql'): # # { mysql datatype from descriptor - # + # if self.debug: logging.debug('') - logging.debug(f'mysql:') + logging.debug('mysql:') logging.debug('col[1]:') logging.debug(col[1]) coltype = int(col[1]) - + if (coltype == 253): dbdatatype = 'VARCHAR' - + if (coltype == 3): dbdatatype = 'LONG' @@ -479,10 +478,10 @@ def __init__(self, cursor, workdir, dd, **kwargs): elif (self.dbms.lower() == 'sqlite3'): # # { mysql datatype from descriptor - # + # if self.debug: logging.debug('') - logging.debug(f'sqlite3:') + logging.debug('sqlite3:') dbdatatype = '' # @@ -495,14 +494,14 @@ def __init__(self, cursor, workdir, dd, **kwargs): logging.debug(f'dbdatatype = {dbdatatype:s}') logging.debug(f'size = {size:d}') - except Exception as e: - logging.debug(f'dbdatatype not set.') + except Exception: + logging.debug('dbdatatype not set.') pass if(col[2] is not None): size = int(col[2]) - + if self.debug: logging.debug(f'size= {size:d}') @@ -562,7 +561,7 @@ def __init__(self, cursor, workdir, dd, **kwargs): try: if len(units) <= 0: units = '' - except Exception as e: + except Exception: units = '' isddcolarr.append(1) @@ -582,10 +581,10 @@ def __init__(self, cursor, workdir, dd, **kwargs): if self.debug: logging.debug('') - logging.debug(f'char type col in dd') + logging.debug('char type col in dd') logging.debug('size=') logging.debug(size) - + if (len(colname) > width): width = len(colname) @@ -671,7 +670,7 @@ def __init__(self, cursor, workdir, dd, **kwargs): width = 80 if ((size is not None) and (size > 0)): width = size - + if(len(colname) > width): width = len(colname) @@ -840,7 +839,7 @@ def __init__(self, cursor, workdir, dd, **kwargs): # { dbdatatype == numeric: this is pgsql oid=1700 special dtype, # if no dd, then we assume it is a double number, although # this dbdatatype is often used to represent a large integer. - # + # coltype = 'double' dbtype = dbdatatype @@ -856,7 +855,7 @@ def __init__(self, cursor, workdir, dd, **kwargs): fmt = str(width) + '.14e' # - # } end dbdatatype == numeric + # } end dbdatatype == numeric # else: @@ -883,12 +882,12 @@ def __init__(self, cursor, workdir, dd, **kwargs): # # - # Special cases: if colname == 'RA' or 'DEC' use + # Special cases: if colname == 'RA' or 'DEC' use # racol, decol in dd # if (dd is not None): - + if(colname.lower() == 'ra'): if((dbdatatype == 'NUMBER') and \ @@ -948,7 +947,7 @@ def __init__(self, cursor, workdir, dd, **kwargs): # to avoid excess data transfer) # - len_arr = len(namearr) + len(namearr) ddlist.append(namearr) ddlist.append(typearr) @@ -984,14 +983,14 @@ def __init__(self, cursor, workdir, dd, **kwargs): try: if self.debug: logging.debug('') - logging.debug(f'Calling writerecs()[1]:') - logging.debug(f'format: ' + str(self.format)) - logging.debug(f'ddlist: ' + str(ddlist)) - logging.debug(f'rowslist: ' + str(rowslist)) - logging.debug(f'ishdr: ' + str(self.ishdr)) - logging.debug(f'coldesc: ' + str(self.coldesc)) - logging.debug(f'overflow: ' + str(self.overflow)) - logging.debug(f'istail: ' + str(self.istail)) + logging.debug('Calling writerecs()[1]:') + logging.debug('format: ' + str(self.format)) + logging.debug('ddlist: ' + str(ddlist)) + logging.debug('rowslist: ' + str(rowslist)) + logging.debug('ishdr: ' + str(self.ishdr)) + logging.debug('coldesc: ' + str(self.coldesc)) + logging.debug('overflow: ' + str(self.overflow)) + logging.debug('istail: ' + str(self.istail)) istatus = writerecs.writerecs(self.outpath, self.format, ddlist, rowslist, self.ishdr, @@ -1074,7 +1073,7 @@ def __init__(self, cursor, workdir, dd, **kwargs): #if (self.dbms.lower() == 'mysql'): # rows = cursor.fetchmany (self.cursor.arraysize) - #else: + #else: # rows = cursor.fetchmany(self.cursor.arraysize) rowslist = [] @@ -1109,14 +1108,14 @@ def __init__(self, cursor, workdir, dd, **kwargs): if self.debug: logging.debug('') - logging.debug(f'Calling writerecs()[2]:') - logging.debug(f'format: ' + str(self.format)) - logging.debug(f'ddlist: ' + str(ddlist)) - logging.debug(f'rowslist: ' + str(rowslist)) - logging.debug(f'ishdr: ' + str(self.ishdr)) - logging.debug(f'coldesc: ' + str(self.coldesc)) - logging.debug(f'overflow: ' + str(self.overflow)) - logging.debug(f'istail: ' + str(self.istail)) + logging.debug('Calling writerecs()[2]:') + logging.debug('format: ' + str(self.format)) + logging.debug('ddlist: ' + str(ddlist)) + logging.debug('rowslist: ' + str(rowslist)) + logging.debug('ishdr: ' + str(self.ishdr)) + logging.debug('coldesc: ' + str(self.coldesc)) + logging.debug('overflow: ' + str(self.overflow)) + logging.debug('istail: ' + str(self.istail)) istatus = writerecs.writerecs(self.outpath, self.format, ddlist, rowslist, self.ishdr, @@ -1139,7 +1138,7 @@ def __init__(self, cursor, workdir, dd, **kwargs): raise Exception(str(e)) if self.debug: - logging.debug (f'return') + logging.debug ('return') logging.debug('') return @@ -1152,12 +1151,12 @@ def __init__(self, cursor, workdir, dd, **kwargs): if self.debug: logging.debug ( \ f'nrec = {nrec:d} and ibatch = {ibatch:d}') - logging.debug (f'break to write last batch of data') + logging.debug ('break to write last batch of data') break # - # } end batch > 0 + # } end batch > 0 # # # } end nrec = 0 @@ -1167,11 +1166,11 @@ def __init__(self, cursor, workdir, dd, **kwargs): # the column "description" block returned by the DBMS # does not give any datatypes. For those cases where # we don't have a priori schema information, we have to - # wait for the first data return to determine types. + # wait for the first data return to determine types. # This is complicated by the possibility that for some # records some columns are sometimes NULL (value 'None'). - # So while we don't want to scan all the data, we have + # So while we don't want to scan all the data, we have # to scan until we find some value for every column. if ((ibatch == 0) and (self.dbms.lower() == 'sqlite')): @@ -1246,7 +1245,7 @@ def __init__(self, cursor, workdir, dd, **kwargs): # # {if pgsql dbtype = 'numeric', dtype will be decimal, # make it integer or float depending on the python type - # of typearr value + # of typearr value # dtype = type (row[i]).__name__ @@ -1292,12 +1291,12 @@ def __init__(self, cursor, workdir, dd, **kwargs): # - # For DBMSs like Oracle that store numbers as generic + # For DBMSs like Oracle that store numbers as generic # NUMBER type, we need to try to distinguish between - # int and double by checking the output row[i]'s + # int and double by checking the output row[i]'s # python datatype -- which might not be completely - # accurate so we check the whole column to make - # summary determination + # accurate so we check the whole column to make + # summary determination # if ((self.dbms.lower() == 'oracle') and \ (ibatch == 0) and \ @@ -1349,7 +1348,7 @@ def __init__(self, cursor, workdir, dd, **kwargs): if self.debug: logging.debug('----------------------------------------') - + # # The following block determines the NUMBER type ORACLE arrays's # datatype -- whether the column should be double or int based @@ -1418,21 +1417,21 @@ def __init__(self, cursor, workdir, dd, **kwargs): if self.debug: logging.debug('') - logging.debug(f'Calling writerecs()[3]:') - logging.debug(f'format: ' + str(self.format)) - logging.debug(f'ddlist: ' + str(ddlist)) - logging.debug(f'rowslist: ' + str(rowslist)) - logging.debug(f'ishdr: ' + str(self.ishdr)) - logging.debug(f'coldesc: ' + str(self.coldesc)) - logging.debug(f'overflow: ' + str(self.overflow)) - logging.debug(f'istail: ' + str(self.istail)) - logging.debug(f'cursor.arraysize' + str(self.cursor.arraysize)) + logging.debug('Calling writerecs()[3]:') + logging.debug('format: ' + str(self.format)) + logging.debug('ddlist: ' + str(ddlist)) + logging.debug('rowslist: ' + str(rowslist)) + logging.debug('ishdr: ' + str(self.ishdr)) + logging.debug('coldesc: ' + str(self.coldesc)) + logging.debug('overflow: ' + str(self.overflow)) + logging.debug('istail: ' + str(self.istail)) + logging.debug('cursor.arraysize' + str(self.cursor.arraysize)) istatus = writerecs.writerecs(self.outpath, self.format, ddlist, rowslist, self.ishdr, self.coldesc, self.overflow, self.istail) - + if self.debug: logging.debug('returned writerecs.writerecs') diff --git a/ctest/ctap.py b/ctest/ctap.py index baed052..76e2f04 100755 --- a/ctest/ctap.py +++ b/ctest/ctap.py @@ -3,10 +3,9 @@ # This example is as much documentation as processing. In a real # application we would only need parts of it. -from TAP.tapstore import tapStore -from TAP.taputil import tapUtil -from TAP.tapquery import tapQuery - +from TAP.tapquery import tapQuery +from TAP.tapstore import tapStore +from TAP.taputil import tapUtil # We may run into problems while connecting configuratio information, # translating the ADQL query into SQL, or submitting the user query @@ -42,7 +41,7 @@ def tapc(catalog, select_stmt, result_tbl): cursor = conn.cursor() cursor.execute('select count(*) from ' + catalog) rowcount = cursor.fetchone()[0] - + # But the main purpose of TAP is to run ADQL queries, including spatial constraints # and output formatting (output file type, column width, and number formatting). @@ -58,7 +57,7 @@ def tapc(catalog, select_stmt, result_tbl): # The returned 'query' object has several internal parameters but the ones most - # useful for further processing are the return status, the output file name + # useful for further processing are the return status, the output file name # (though this is just the name we put in above), and the return record count. if status == 'OK': @@ -66,5 +65,5 @@ def tapc(catalog, select_stmt, result_tbl): else: return status, msg, 0, 0 - except Exception as e: + except Exception: return 'ERROR', msg, 0, 0 diff --git a/ctest/dbtap.py b/ctest/dbtap.py index 84f5777..2fcfa19 100644 --- a/ctest/dbtap.py +++ b/ctest/dbtap.py @@ -1,6 +1,6 @@ -from TAP.tapstore import tapStore -from TAP.taputil import tapUtil -from TAP.tapquery import tapQuery +from TAP.tapquery import tapQuery +from TAP.tapstore import tapStore +from TAP.taputil import tapUtil dbutils = None dbconfig = None @@ -20,9 +20,9 @@ def dbinit(): dbconn = dbstore.getConn() dbadql = dbstore.getADQL() - except Exception as e: + except Exception: return 0 - + return 1 @@ -38,13 +38,13 @@ def dbquery(adql_string, filename): try: format = 'ipac' - cursor = dbconn.cursor() - + dbconn.cursor() + sql_string = dbadql.sql(adql_string) dbQuery = tapQuery(connectInfo=dbconfig.connectInfo, - conn=dbconn, - query=sql_string, + conn=dbconn, + query=sql_string, filename=filename, format=format) @@ -59,6 +59,6 @@ def dbquery(adql_string, filename): else: return status, msg, 0 - except Exception as e: + except Exception: return status, msg, 0 diff --git a/load_schema/load_schema.py b/load_schema/load_schema.py index 2f1a879..b16d782 100644 --- a/load_schema/load_schema.py +++ b/load_schema/load_schema.py @@ -1,7 +1,7 @@ #!/bin/env python -import sqlite3 import pprint +import sqlite3 pp = pprint.PrettyPrinter() @@ -29,11 +29,11 @@ # worked out once; from here on if we were to add tables we would just need # to add one "tables" record and its "columns" records. -# Finally, there are two more TAP_SCHEMA tables describing how foreign keys +# Finally, there are two more TAP_SCHEMA tables describing how foreign keys # in our tables relate the tables to each other (i.e., allow JOINS). Our one # real data table has no such keys but all these TAP_SCHEMA tables themselves # do have such relationships (and so must be included). However, unless we add -# new data tables which can join with each other, we never need to touch these +# new data tables which can join with each other, we never need to touch these # two tables again. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..c952242 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,32 @@ +[tool.pytest.ini_options] +testpaths = ["tests"] +addopts = "-ra" + +[tool.ruff] +line-length = 120 +target-version = "py39" +exclude = [ + "build", + "dist", + "docs", + "Sphinx", # Sphinx config conventionally puts imports below comments + "Jupyter", + "tests/fixtures", +] + +[tool.ruff.lint] +select = ["E", "F", "W", "I"] +ignore = [ + # Existing code style choices we're not touching in this pass: + "E501", # line too long — many long log strings and SQL literals + "E701", # multiple statements on one line — common in if-on-one-line style + "E711", # comparison to None — code uses `== None` extensively + "E712", # comparison to True/False — code uses `== True` extensively + "E722", # bare except — code uses bare except in many places + "E741", # ambiguous variable name — `l` used in several loops + # Pre-existing real bugs to address in a follow-up; intentionally not + # fixed in the CI-setup PR to keep the diff focused. + # See: TAP/taputil.py:221,223,227,246,251,261 (SQLite + MySQL branches) + # TAP/tap.py:2402 (`result` typo in a debug log path) + "F821", # undefined-name +] diff --git a/requirements-test.txt b/requirements-test.txt new file mode 100644 index 0000000..c734ed5 --- /dev/null +++ b/requirements-test.txt @@ -0,0 +1,4 @@ +pytest>=7 +ruff>=0.1 +requests +configobj diff --git a/setup.py b/setup.py index afee75a..ac52e50 100644 --- a/setup.py +++ b/setup.py @@ -1,16 +1,16 @@ -import os -from setuptools import setup from distutils.extension import Extension -setup(name='nexsciTAP', +from setuptools import setup + +setup(name='nexsciTAP', version='3.0.0', - author='John Good', - author_email='jcg@caltech.edu', - license='LICENSE', - keywords='astronomy database ADQL web-service', + author='John Good', + author_email='jcg@caltech.edu', + license='LICENSE', + keywords='astronomy database ADQL web-service', url = 'https://github.com/Caltech-IPAC/nexsciTAP', - description='NExScI VO Table Access Protocol (TAP) web service', + description='NExScI VO Table Access Protocol (TAP) web service', long_description=open('README.md').read(), ext_modules=[Extension('TAP/writerecs', ['TAP/writerecsmodule.c'])], install_requires=['ADQL', 'spatial_index', 'configobj'], diff --git a/tests/_helpers.py b/tests/_helpers.py new file mode 100644 index 0000000..11873ce --- /dev/null +++ b/tests/_helpers.py @@ -0,0 +1,22 @@ +"""Shared test utility functions. Not auto-discovered by pytest (no `test_` prefix).""" +from __future__ import annotations + + +def tap_sync_url(server: str) -> str: + """The TAP sync endpoint URL relative to *server*.""" + return f"{server}/cgi-bin/TAP/nph-tap.py/sync" + + +def tap_async_url(server: str) -> str: + """The TAP async endpoint URL relative to *server*.""" + return f"{server}/cgi-bin/TAP/nph-tap.py/async" + + +def row_count_from_ipac(body: str) -> int: + """Count data rows in an IPAC ASCII table body. + + IPAC format has 4 header lines (names, types, units, nulls) followed by + data rows. Empty trailing lines are ignored. + """ + lines = [L for L in body.splitlines() if L.strip()] + return max(0, len(lines) - 4) diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..0974852 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,202 @@ +"""Shared pytest fixtures. + +Spins up a per-session HTTP server backed by SQLite fixtures and the +production CGI shim, so tests can exercise the full HTTP→CGI→TAP→DB pipeline. + +Layout under the session tmpdir: + + / + tap_schema.db — populated TAP_SCHEMA + test_data.db — data_l0, data_l1, test_access, test_users + TAP.conf — substituted from fixtures/TAP.conf.template + workdir/ — TAP_WORKDIR (request artifacts) + cgi-bin/ + TAP/ + nph-tap.py — copy of fixtures/nph-tap.py +""" +from __future__ import annotations + +import http.server +import os +import shutil +import socket +import sys +import threading +import time +from pathlib import Path + +import pytest + +TESTS_DIR = Path(__file__).parent +FIXTURES = TESTS_DIR / "fixtures" +STUBS = FIXTURES / "stubs" +sys.path.insert(0, str(FIXTURES)) +sys.path.insert(0, str(TESTS_DIR)) +sys.path.insert(0, str(STUBS)) + +import build_test_db # noqa: E402 + + +def _free_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +@pytest.fixture(scope="session") +def fixture_root(tmp_path_factory) -> Path: + """Build all the per-session artifacts and return the tmpdir root.""" + root = tmp_path_factory.mktemp("nexscitap-ci") + + build_test_db.build_all(str(root)) + + (root / "workdir").mkdir() + cgi_dir = root / "cgi-bin" / "TAP" + cgi_dir.mkdir(parents=True) + shutil.copy(FIXTURES / "nph-tap.py", cgi_dir / "nph-tap.py") + (cgi_dir / "nph-tap.py").chmod(0o755) + + return root + + +@pytest.fixture(scope="session") +def tap_conf(fixture_root: Path) -> Path: + """Materialize TAP.conf with absolute paths substituted in.""" + port = _free_port() + template = (FIXTURES / "TAP.conf.template").read_text() + conf = template.format( + TEST_WORKDIR=str(fixture_root / "workdir"), + TEST_HTTP_URL=f"http://127.0.0.1:{port}", + TEST_DB_PATH=str(fixture_root / "test_data.db"), + TEST_TAP_SCHEMA=str(fixture_root / "tap_schema.db"), + ) + conf_path = fixture_root / "TAP.conf" + conf_path.write_text(conf) + # Stash the port so tap_server can reuse it + (fixture_root / ".port").write_text(str(port)) + return conf_path + + +class _NphCGIHandler(http.server.BaseHTTPRequestHandler): + """Minimal nph-CGI runner. + + The TAP CGI script writes its own HTTP status line (it's an `nph-` / + non-parsed-headers script in Apache parlance — see TAP/tap.py:755, 2538, + etc.). Python's stdlib CGIHTTPRequestHandler is not nph-aware: it always + prepends a `HTTP/1.1 200 Script output follows` status line, so the + script's status line ends up looking like a header to the client. + + This handler runs the CGI subprocess and forwards its raw stdout to the + socket without modification — exactly what an Apache nph- script does. + """ + + nph_script_path = "" # set by the tap_server fixture before serve_forever + + def do_GET(self): + self._run_cgi() + + def do_POST(self): + self._run_cgi() + + def _run_cgi(self): + # Parse PATH_INFO and QUERY_STRING from self.path. + from urllib.parse import urlsplit + + parts = urlsplit(self.path) + # Anything after the script mount point is PATH_INFO. + # (Our tests use /cgi-bin/TAP/nph-tap.py/sync etc.) + path = parts.path + marker = "/nph-tap.py" + if marker in path: + path_info = path.split(marker, 1)[1] + else: + path_info = "" + + # Read request body (if any) for the subprocess stdin. + content_length = int(self.headers.get("Content-Length", "0") or "0") + body = self.rfile.read(content_length) if content_length else b"" + + env = os.environ.copy() + env.update({ + "GATEWAY_INTERFACE": "CGI/1.1", + "SERVER_PROTOCOL": "HTTP/1.1", + "SERVER_SOFTWARE": "test-nph-cgi/1.0", + "REQUEST_METHOD": self.command, + "QUERY_STRING": parts.query, + "PATH_INFO": path_info, + "SCRIPT_NAME": "/cgi-bin/TAP/nph-tap.py", + "REMOTE_ADDR": self.client_address[0], + "CONTENT_TYPE": self.headers.get("Content-Type", ""), + "CONTENT_LENGTH": str(content_length), + "HTTP_HOST": self.headers.get("Host", ""), + }) + + import subprocess + proc = subprocess.Popen( + [sys.executable, self.server.nph_script_path], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env, + ) + stdout, stderr = proc.communicate(input=body) + + # Forward raw subprocess stdout to the socket. The script is + # responsible for emitting a valid HTTP status line + headers. + try: + self.wfile.write(stdout) + self.wfile.flush() + except BrokenPipeError: + pass + + # Quieter logging during test runs. + def log_message(self, format, *args): + return + + +@pytest.fixture(scope="session") +def tap_server(fixture_root: Path, tap_conf: Path): + """Start http.server with the CGI handler. Yields the base URL.""" + port = int((fixture_root / ".port").read_text()) + + os.environ["TAP_CONF"] = str(tap_conf) + os.environ["TAP_PORT"] = str(port) + + # CGIHTTPRequestHandler runs each request as a subprocess; it inherits + # the parent's environment but not the in-process sys.path. Add the + # repo root to PYTHONPATH so `from TAP import tap` works in the CGI + # subprocess even if the package isn't pip-installed (CI installs the + # package; local pytest may not). + repo_root = TESTS_DIR.parent + paths = [str(repo_root), str(STUBS)] + existing = os.environ.get("PYTHONPATH", "") + if existing: + paths.append(existing) + os.environ["PYTHONPATH"] = os.pathsep.join(paths) + + original_cwd = os.getcwd() + os.chdir(fixture_root) + + httpd = http.server.HTTPServer(("127.0.0.1", port), _NphCGIHandler) + httpd.allow_reuse_address = True + httpd.nph_script_path = str(fixture_root / "cgi-bin" / "TAP" / "nph-tap.py") + thread = threading.Thread(target=httpd.serve_forever, daemon=True) + thread.start() + + # Brief wait for the server to come up + for _ in range(50): + try: + with socket.create_connection(("127.0.0.1", port), timeout=0.2): + break + except OSError: + time.sleep(0.05) + + base_url = f"http://127.0.0.1:{port}" + try: + yield base_url + finally: + httpd.shutdown() + httpd.server_close() + os.chdir(original_cwd) + + diff --git a/tests/fixtures/TAP.conf.template b/tests/fixtures/TAP.conf.template new file mode 100644 index 0000000..e0280d2 --- /dev/null +++ b/tests/fixtures/TAP.conf.template @@ -0,0 +1,45 @@ +# TAP.conf template for CI tests. +# +# The pytest `tap_conf` fixture in tests/conftest.py substitutes the +# placeholders below with absolute paths into the per-session tmpdir. +# +# Placeholders: +# {TEST_WORKDIR} — working dir for TAP request artifacts +# {TEST_HTTP_URL} — http://localhost:NNNN (fixture-allocated) +# {TEST_DB_PATH} — absolute path to test_data.db +# {TEST_TAP_SCHEMA} — absolute path to tap_schema.db + +[WEB] + TAP_WORKDIR={TEST_WORKDIR} + TAP_WORKURL=/workspace + + HTTP_URL={TEST_HTTP_URL} + HTTP_PORT=0 + CGI_PGM=TAP + + +# Default configuration instance — no `instance=` HTTP parameter needed. +[DBMS] + DBMS=sqlite3 + DB={TEST_DB_PATH} + TAP_SCHEMA={TEST_TAP_SCHEMA} + TAP_SCHEMA_FILE=TAP_SCHEMA + + # Propfilter setup — exercises the propflag guard added in PR #17. + ACCESS_TBL=test_access + USERS_TBL=test_users + FILEID=l0filename + ACCESSID=program + PROPFILTER=neid + RACOL=ra + DECCOL=dec + + +[SPTIND] + MODE=HTM + LEVEL=20 + XCOL=x + YCOL=y + ZCOL=z + COLNAME=sptind + ENCODING=BASE10 diff --git a/tests/fixtures/build_test_db.py b/tests/fixtures/build_test_db.py new file mode 100644 index 0000000..caa40ab --- /dev/null +++ b/tests/fixtures/build_test_db.py @@ -0,0 +1,344 @@ +"""Build the SQLite test fixtures: tap_schema.db and test_data.db. + +Used by the pytest `test_db` session fixture in tests/conftest.py. Can also +be run standalone for local inspection: + + python tests/fixtures/build_test_db.py /tmp/fixture-out +""" +from __future__ import annotations + +import os +import sqlite3 +import sys +from datetime import datetime, timedelta + + +# ---------------------------------------------------------------- TAP_SCHEMA + +TAP_SCHEMA_DDL = [ + """ + CREATE TABLE IF NOT EXISTS schemas ( + schema_name TEXT NOT NULL, + utype TEXT, + description TEXT, + schema_index INTEGER, + PRIMARY KEY (schema_name) + ) + """, + """ + CREATE TABLE IF NOT EXISTS tables ( + schema_name TEXT NOT NULL, + table_name TEXT NOT NULL, + table_type TEXT, + utype TEXT, + description TEXT, + table_index INTEGER, + PRIMARY KEY (table_name) + ) + """, + """ + CREATE TABLE IF NOT EXISTS columns ( + table_name TEXT NOT NULL, + column_name TEXT NOT NULL, + datatype TEXT, + arraysize TEXT, + size INTEGER, + description TEXT, + utype TEXT, + unit TEXT, + ucd TEXT, + indexed INTEGER, + principal INTEGER, + std INTEGER, + column_index INTEGER, + PRIMARY KEY (table_name, column_name) + ) + """, + """ + CREATE TABLE IF NOT EXISTS keys ( + key_id TEXT NOT NULL, + from_table TEXT, + target_table TEXT, + description TEXT, + utype TEXT, + PRIMARY KEY (key_id) + ) + """, + """ + CREATE TABLE IF NOT EXISTS key_columns ( + key_id TEXT NOT NULL, + from_column TEXT, + target_column TEXT, + PRIMARY KEY (key_id, from_column) + ) + """, +] + +SCHEMAS = [ + ("test", "Test data for nexsciTAP CI", 1), + ("TAP_SCHEMA", "Standard TAP schema description.", 2), +] + +# Only data_l0 and data_l1 are reachable as query targets; test_access and +# test_users are propfilter-internal tables that the validator allows in +# subqueries by way of the ACCESS_TBL/USERS_TBL config entries. +TABLES = [ + # (schema, table, description) + ("test", "data_l0", "Public-data test table (propint=0)."), + ("test", "data_l1", "Proprietary test table (mix of embargoed and released)."), + ("test", "test_access", "Propfilter access table — internal."), + ("test", "test_users", "Propfilter users table — internal."), + ("TAP_SCHEMA", "schemas", "TAP_SCHEMA.schemas."), + ("TAP_SCHEMA", "tables", "TAP_SCHEMA.tables."), + ("TAP_SCHEMA", "columns", "TAP_SCHEMA.columns."), + ("TAP_SCHEMA", "keys", "TAP_SCHEMA.keys."), + ("TAP_SCHEMA", "key_columns", "TAP_SCHEMA.key_columns."), +] + +# Columns for data_l0 and data_l1 — shape mirrors NEID's neidl0/neidl1 enough +# to drive the propfilter code paths and the UI's CASE-WHEN download-link +# query pattern. Keep small; only the columns used by tests need to exist. +DATA_L0_COLUMNS = [ + ("data_l0", "obsdate", "char", "DATE", "Observation date."), + ("data_l0", "obsjd", "double", None, "Julian date of observation."), + ("data_l0", "program", "char", None, "Program ID."), + ("data_l0", "piname", "char", None, "PI name."), + ("data_l0", "obsmode", "char", None, "Observation mode."), + ("data_l0", "obstype", "char", None, "Sci or Cal."), + ("data_l0", "l0propint", "int", "months", "Proprietary interval (months)."), + ("data_l0", "l0filename", "char", None, "Filename."), + ("data_l0", "l0filehand", "char", None, "File handle."), + ("data_l0", "l0filepath", "char", None, "File path."), + ("data_l0", "object", "char", None, "Object name."), + ("data_l0", "qobject", "char", None, "Resolved object."), + ("data_l0", "qra", "char", "deg", "Query RA."), + ("data_l0", "qdec", "char", "deg", "Query Dec."), + ("data_l0", "ra", "double", "deg", "RA."), + ("data_l0", "dec", "double", "deg", "Dec."), +] +DATA_L1_COLUMNS = [(("data_l1",) + c[1:]) for c in DATA_L0_COLUMNS] +# Rename the l0-prefixed columns to l1-prefixed for data_l1 +DATA_L1_COLUMNS = [ + (t, name.replace("l0", "l1") if name.startswith("l0") else name, *rest) + for (t, name, *rest) in DATA_L1_COLUMNS +] + +# Minimal column metadata for the propfilter-internal and TAP_SCHEMA tables +# — TableValidator looks at table names not column names, but the data +# dictionary code path queries `select * from tap_schema.columns where +# table_name = ?` so the rows must exist. +ACCESS_COLUMNS = [ + ("test_access", "userid", "char", None, "User ID."), + ("test_access", "program", "char", None, "Program ID accessible to userid."), +] +USERS_COLUMNS = [ + ("test_users", "userid", "char", None, "User ID."), + ("test_users", "cookie", "char", None, "Session cookie."), +] +SELF_COLUMNS = [ + ("schemas", "schema_name", "char", None, ""), + ("tables", "schema_name", "char", None, ""), + ("tables", "table_name", "char", None, ""), + ("columns", "table_name", "char", None, ""), + ("columns", "column_name", "char", None, ""), + ("columns", "datatype", "char", None, ""), + ("columns", "unit", "char", None, ""), + ("columns", "description", "char", None, ""), +] + + +def build_tap_schema(path: str) -> None: + """Populate tap_schema.db at *path*.""" + if os.path.exists(path): + os.remove(path) + conn = sqlite3.connect(path) + cur = conn.cursor() + for ddl in TAP_SCHEMA_DDL: + cur.execute(ddl) + cur.executemany( + "INSERT INTO schemas (schema_name, description, schema_index) VALUES (?, ?, ?)", + SCHEMAS, + ) + # TableValidator expects table_name values to be schema-qualified + # (e.g. "TAP_SCHEMA.tables", not just "tables") so it can populate its + # allowed_schemas set from the dotted prefix. See tablevalidator.py:50-60. + qualified_tables = [ + (schema, f"{schema}.{table}", description) + for (schema, table, description) in TABLES + ] + cur.executemany( + "INSERT INTO tables (schema_name, table_name, description) VALUES (?, ?, ?)", + qualified_tables, + ) + all_cols = DATA_L0_COLUMNS + DATA_L1_COLUMNS + ACCESS_COLUMNS + USERS_COLUMNS + SELF_COLUMNS + cur.executemany( + """INSERT INTO columns (table_name, column_name, datatype, unit, description) + VALUES (?, ?, ?, ?, ?)""", + all_cols, + ) + conn.commit() + conn.close() + + +# ---------------------------------------------------------------- data tables + +def build_test_data(path: str) -> None: + """Populate test_data.db with the data_l0, data_l1, test_access, test_users tables.""" + if os.path.exists(path): + os.remove(path) + conn = sqlite3.connect(path) + cur = conn.cursor() + + # data_l0 — public (l0propint = 0). 6 rows across 2 programs. + cur.execute(""" + CREATE TABLE data_l0 ( + obsdate TEXT, + obsjd REAL, + program TEXT, + piname TEXT, + obsmode TEXT, + obstype TEXT, + l0propint INTEGER, + l0filename TEXT, + l0filehand TEXT, + l0filepath TEXT, + object TEXT, + qobject TEXT, + qra TEXT, + qdec TEXT, + ra REAL, + dec REAL + ) + """) + + now = datetime.now() + l0_rows = [] + for i in range(6): + d = now - timedelta(days=10 + i) + prog = "TEST-A" if i % 2 == 0 else "TEST-B" + l0_rows.append(( + d.strftime("%Y-%m-%d %H:%M:%S"), # obsdate + 2460000.0 + i, # obsjd + prog, # program + "Test PI", # piname + "hr", # obsmode + "Cal" if i < 2 else "Sci", # obstype + 0, # l0propint — PUBLIC + f"testL0_2026{i:04d}.fits", # l0filename + f"/handle/{i}", # l0filehand + f"/data/L0/testL0_2026{i:04d}.fits", # l0filepath + f"OBJ-{i}", # object + f"OBJ-{i}", # qobject + "10.0", # qra + "20.0", # qdec + 10.0, # ra + 20.0, # dec + )) + cur.executemany( + "INSERT INTO data_l0 VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", l0_rows + ) + + # data_l1 — proprietary mix. Critical fixture invariant: at least some + # rows must be STILL EMBARGOED (today < add_months(obsdate, l1propint)) + # so propfilter has something to filter out. Without this, propflag=0 + # vs propflag=1 returns identical counts and the security test can't + # detect a regression. + cur.execute(""" + CREATE TABLE data_l1 ( + obsdate TEXT, + obsjd REAL, + program TEXT, + piname TEXT, + obsmode TEXT, + obstype TEXT, + l1propint INTEGER, + l1filename TEXT, + l1filehand TEXT, + l1filepath TEXT, + object TEXT, + qobject TEXT, + qra TEXT, + qdec TEXT, + ra REAL, + dec REAL + ) + """) + l1_rows = [] + # 3 public rows (propint=0) + for i in range(3): + d = now - timedelta(days=30 + i) + l1_rows.append(( + d.strftime("%Y-%m-%d %H:%M:%S"), 2460500.0 + i, + "TEST-PUBLIC", "PI Public", "hr", "Sci", 0, + f"testL1_public_{i:02d}.fits", f"/h/p/{i}", f"/data/L1/p{i}.fits", + "STAR", "STAR", "10.0", "20.0", 10.0, 20.0, + )) + # 3 expired-embargo rows (propint=12 months, obsdate > 2 years ago) + for i in range(3): + d = now - timedelta(days=900 + i) + l1_rows.append(( + d.strftime("%Y-%m-%d %H:%M:%S"), 2459500.0 + i, + "TEST-EXPIRED", "PI Expired", "hr", "Sci", 12, + f"testL1_expired_{i:02d}.fits", f"/h/e/{i}", f"/data/L1/e{i}.fits", + "STAR", "STAR", "10.0", "20.0", 10.0, 20.0, + )) + # 4 STILL-EMBARGOED rows (propint=24 months, obsdate < 6 months ago) + for i in range(4): + d = now - timedelta(days=60 + i * 10) + l1_rows.append(( + d.strftime("%Y-%m-%d %H:%M:%S"), 2461000.0 + i, + "TEST-PROPRIETARY", "PI Proprietary", "hr", "Sci", 24, + f"testL1_embargoed_{i:02d}.fits", f"/h/x/{i}", f"/data/L1/x{i}.fits", + "STAR", "STAR", "10.0", "20.0", 10.0, 20.0, + )) + cur.executemany( + "INSERT INTO data_l1 VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", l1_rows + ) + + # test_access — propfilter access table. Maps userid -> program. + cur.execute(""" + CREATE TABLE test_access ( + userid TEXT, + program TEXT + ) + """) + cur.executemany( + "INSERT INTO test_access VALUES (?, ?)", + [ + ("testuser", "TEST-PROPRIETARY"), + ], + ) + + # test_users — propfilter users table. Holds session token mapping. + cur.execute(""" + CREATE TABLE test_users ( + userid TEXT, + cookie TEXT + ) + """) + cur.executemany( + "INSERT INTO test_users VALUES (?, ?)", + [ + ("testuser", "fake-test-cookie"), + ], + ) + + conn.commit() + conn.close() + + +def build_all(out_dir: str) -> dict: + """Build both fixtures under *out_dir*. Returns {'tap_schema': path, 'test_data': path}.""" + os.makedirs(out_dir, exist_ok=True) + tap_schema_path = os.path.join(out_dir, "tap_schema.db") + test_data_path = os.path.join(out_dir, "test_data.db") + build_tap_schema(tap_schema_path) + build_test_data(test_data_path) + return {"tap_schema": tap_schema_path, "test_data": test_data_path} + + +if __name__ == "__main__": + out = sys.argv[1] if len(sys.argv) > 1 else "." + paths = build_all(out) + for k, v in paths.items(): + print(f"{k}: {v}") diff --git a/tests/fixtures/nph-tap.py b/tests/fixtures/nph-tap.py new file mode 100644 index 0000000..3912999 --- /dev/null +++ b/tests/fixtures/nph-tap.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python +"""CGI entry point for the CI test fixture. + +Matches the production CGI shim documented at Sphinx/install.rst. The pytest +`tap_server` fixture in tests/conftest.py serves this file via +http.server.CGIHTTPRequestHandler with TAP_CONF set to the fixture's TAP.conf. +""" +from TAP import tap + +tap.Tap() diff --git a/tests/fixtures/stubs/spatial_index.py b/tests/fixtures/stubs/spatial_index.py new file mode 100644 index 0000000..356c803 --- /dev/null +++ b/tests/fixtures/stubs/spatial_index.py @@ -0,0 +1,22 @@ +"""Minimal stub of the spatial_index package for CI use. + +The real `spatial_index` package on PyPI only ships wheels for Python ≤ 3.8, +so the CI matrix (3.9 / 3.10 / 3.11 / 3.12) can't install it. TAP imports +the package unconditionally for four module-level constants — that's all +the surface we exercise in tests, since no test query uses spatial +geometry functions. + +If/when a test starts exercising spatial queries (POINT, CIRCLE, CONTAINS, +DISTANCE), this stub will need to grow or the real package needs to be +installed. + +Tracking the upstream issue: spatial-index 1.1.0 on PyPI lacks a source +distribution and modern-Python wheels. +""" + + +class SpatialIndex: + HTM = 1 + HPX = 2 + BASE4 = 10 + BASE10 = 16 diff --git a/tests/test_http_endpoint.py b/tests/test_http_endpoint.py new file mode 100644 index 0000000..6fd9377 --- /dev/null +++ b/tests/test_http_endpoint.py @@ -0,0 +1,89 @@ +"""Smoke tests for the CI HTTP fixture. + +These don't probe security boundaries (see test_security_e2e.py in the next +PR). They just confirm the fixture itself works: the CGI launches, the SQLite +DBs are wired in, and a few representative queries make it through the full +HTTP→CGI→TAP→SQLite→IPAC pipeline. +""" +from __future__ import annotations + +import requests +from _helpers import row_count_from_ipac, tap_sync_url + + +def test_server_starts(tap_server: str): + """The fixture comes up and the CGI returns something for a trivial GET.""" + r = requests.get(tap_server) + # Root path is not a CGI script — we expect a 200 with the directory + # listing or a 404; either confirms the server itself is alive. + assert r.status_code in (200, 404) + + +def test_sync_select_l0(tap_server: str): + """A trivial select against data_l0 returns 6 rows in IPAC format.""" + r = requests.post( + tap_sync_url(tap_server), + data={ + "query": "select l0filename, program from data_l0", + "format": "ipac", + }, + ) + assert r.status_code == 200, r.text[:500] + assert row_count_from_ipac(r.text) == 6 + + +def test_sync_select_l1(tap_server: str): + """data_l1 has 10 rows total (3 public + 3 expired + 4 embargoed).""" + r = requests.post( + tap_sync_url(tap_server), + data={ + "query": "select l1filename from data_l1", + "format": "ipac", + # propflag=0 → tapQuery path → no filter → all rows + "propflag": "0", + }, + ) + assert r.status_code == 200, r.text[:500] + assert row_count_from_ipac(r.text) == 10 + + +def test_sync_get_method(tap_server: str): + """Sync TAP must also accept GET (not just POST).""" + r = requests.get( + tap_sync_url(tap_server), + params={ + "query": "select l0filename from data_l0", + "format": "ipac", + }, + ) + assert r.status_code == 200, r.text[:500] + assert row_count_from_ipac(r.text) == 6 + + +def test_tap_schema_query(tap_server: str): + """tap_schema queries take a separate code path; confirm it works.""" + r = requests.post( + tap_sync_url(tap_server), + data={ + "query": "select table_name from tap_schema.tables", + "format": "ipac", + }, + ) + assert r.status_code == 200, r.text[:500] + # Our fixture registers 4 user tables + 5 TAP_SCHEMA self-tables = 9. + assert row_count_from_ipac(r.text) >= 9 + + +def test_csv_format(tap_server: str): + """The CSV serializer works.""" + r = requests.post( + tap_sync_url(tap_server), + data={ + "query": "select l0filename from data_l0", + "format": "csv", + }, + ) + assert r.status_code == 200, r.text[:500] + lines = [L for L in r.text.splitlines() if L.strip()] + # 1 header line + 6 data lines + assert len(lines) == 7 diff --git a/tests/test_tablevalidator.py b/tests/test_tablevalidator.py index 5faff1e..d91719f 100644 --- a/tests/test_tablevalidator.py +++ b/tests/test_tablevalidator.py @@ -3,7 +3,7 @@ import tempfile import unittest -from TAP.tablevalidator import TableValidator, TableValidationError +from TAP.tablevalidator import TableValidationError, TableValidator def _make_db(table_names, schema_name='TAP_SCHEMA', tables_table='tables'): From 1436c9fbcabadb4fac9391a7d75d702bf4ecddae Mon Sep 17 00:00:00 2001 From: BJ Fulton Date: Wed, 20 May 2026 10:02:21 -0600 Subject: [PATCH 03/31] CI: install nexsciTAP and ADQL with --no-deps to work around spatial_index MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The wheel-build verification and the test-job install both failed on the first CI run because pip can't resolve spatial_index on Python 3.9+: the PyPI package only ships wheels for 2.7 / 3.6 / 3.7 / 3.8 and has no source distribution. Workaround in CI: - Test job: install nexsciTAP and ADQL with --no-deps. The test fixture stub in tests/fixtures/stubs/spatial_index.py supplies the four module-level constants TAP uses at import time. configobj (the only other actually-needed runtime dep) is in requirements-test.txt. - Wheel-platforms job: drop the post-build pip install; instead verify the .so artifact is present in the wheel. The wheel itself builds fine — the issue is only at install-resolve time. This is purely a CI workaround. The deeper fix is upstream in spatial_index (publish modern wheels or an sdist). Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/ci.yml | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0e5a578..61c8e0e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,10 +22,18 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install package and test deps + # `spatial_index` on PyPI has no wheels for Python 3.9+ and no + # source distribution, so `pip install .` cannot resolve it on the + # modern matrix. Install nexsciTAP with --no-deps and rely on the + # test fixture's spatial_index stub (tests/fixtures/stubs/) on + # PYTHONPATH for the constants TAP imports. configobj is the only + # actually-needed runtime dep and is in requirements-test.txt. + # ADQL is installed with --no-deps for the same reason. run: | pip install --upgrade pip pip install -r requirements-test.txt - pip install . + pip install --no-deps "ADQL==1.0.6" + pip install --no-deps . - name: Lint with ruff run: ruff check . @@ -54,7 +62,18 @@ jobs: python -m build --wheel - name: Verify wheel artifact + # See test job for spatial_index PyPI situation. We can't install + # the wheel's full deps on modern Python; just confirm the wheel + # was built and includes the C extension. run: | ls dist/ - python -m pip install dist/*.whl - python -c "import TAP; print('TAP imports OK')" + python -c " + import zipfile, glob + wheel = glob.glob('dist/*.whl')[0] + with zipfile.ZipFile(wheel) as z: + names = z.namelist() + assert any('writerecs' in n and n.endswith('.so') for n in names), \ + f'C extension missing from wheel: {names}' + print(f'wheel OK: {wheel}') + print('contains C extension') + " From 0031821d48b267138d91bc6371f754b8a02d77fc Mon Sep 17 00:00:00 2001 From: BJ Fulton Date: Wed, 20 May 2026 10:04:04 -0600 Subject: [PATCH 04/31] CI: add sqlparse, astropy, bs4, xmltodict to test deps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Surfaced by the second CI run: TAP/tablenames.py imports sqlparse, TAP/datadictionary.py imports astropy.io.ascii, and other TAP modules import bs4 and xmltodict. None of these are declared in setup.py's install_requires (a pre-existing bug — declaring them in install_requires is a separate cleanup since the CI install path uses --no-deps anyway). Add them to requirements-test.txt so CI installs the actual runtime surface needed to run the tests. Co-Authored-By: Claude Opus 4.7 (1M context) --- requirements-test.txt | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/requirements-test.txt b/requirements-test.txt index c734ed5..20f2cdf 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,4 +1,13 @@ pytest>=7 ruff>=0.1 requests + +# Runtime deps. These are not declared in setup.py's install_requires (a +# separate cleanup) but the package imports them. The CI install uses +# --no-deps for nexsciTAP itself (to avoid the spatial_index PyPI hole), +# so we list everything actually-needed here. configobj +sqlparse +astropy +beautifulsoup4 +xmltodict From 669983f9fbf3e49f571babd765acfca4a6e502f2 Mon Sep 17 00:00:00 2001 From: BJ Fulton Date: Wed, 20 May 2026 10:24:03 -0600 Subject: [PATCH 05/31] Add security regression tests + pyNEID compat scaffolding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 3+4 of the CI plan. Each test in test_security_e2e.py is named after the specific regression or security boundary it guards, so any future change that re-opens one of those holes turns the corresponding test red at PR time. What's tested (12 active, 1 skipped): - Table whitelist: rejects sqlite_master, rejects unregistered tables, allows propfilter internal tables (test_access) in subqueries - Table extractor: ORDER BY not misidentified, UNKNOWN-statement-skip bypass closed (Oracle ROWNUM) - propflag: zero honored on L0, complex CASE WHEN with subquery FROM does not misfire the whitelist - Statement validator: DML/DDL rejected, dangerous Oracle functions rejected, semicolon-chained statements rejected - HTTP status codes: 400 for query errors, error bodies don't leak internal state (Traceback, paths, Password=) Skipped with detailed reasons in the test bodies: - test_propflag_bypass_routes_l1_through_propfilter — configparam.py only reads PROPFILTER/ACCESS_TBL/USERS_TBL inside its `if dbms == 'oracle'` branch, so on SQLite the propfilter config is silently dropped and the guard becomes a no-op. End-to-end propflag coverage needs the Oracle/Postgres CI matrix from the deferred plan work. pyNEID compat (test_pyneid_compat.py) — module-level skip. pyNEID's query_adql uses the async TAP endpoint (POST + status URL polling + results fetch); the test fixture handler doesn't yet model the per-request workspace lifecycle Apache provides in production. Direct-HTTP coverage of the same code paths is in test_security_e2e.py. Conftest: handler now routes both /cgi-bin/TAP/nph-tap.py/ (direct CGI URL) and /TAP/ (production short form) to the same nph-tap.py script — wires up the URL shape pyNEID expects, even though pyNEID itself remains skipped pending async work. Misalignment surfaced: TAP returns 400 for DML / dangerous-function / semicolon-chained rejections (validate_statement raises plain Exception), but 403 for system-catalog rejections (raises TableValidationError). These are all security-policy decisions and should arguably all be 403. Tests match current behavior; harmonizing is a follow-up. Co-Authored-By: Claude Opus 4.7 (1M context) --- requirements-test.txt | 5 + tests/conftest.py | 10 +- tests/test_pyneid_compat.py | 54 ++++++++ tests/test_security_e2e.py | 252 ++++++++++++++++++++++++++++++++++++ 4 files changed, 319 insertions(+), 2 deletions(-) create mode 100644 tests/test_pyneid_compat.py create mode 100644 tests/test_security_e2e.py diff --git a/requirements-test.txt b/requirements-test.txt index 20f2cdf..343732f 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -11,3 +11,8 @@ sqlparse astropy beautifulsoup4 xmltodict + +# pyneid is not currently used — tests/test_pyneid_compat.py is skipped +# at module level until async-flow fixture support lands. Listed as a +# reminder; comment back in when re-enabling. +# pyneid diff --git a/tests/conftest.py b/tests/conftest.py index 0974852..6aa0cf6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -103,12 +103,18 @@ def _run_cgi(self): from urllib.parse import urlsplit parts = urlsplit(self.path) - # Anything after the script mount point is PATH_INFO. - # (Our tests use /cgi-bin/TAP/nph-tap.py/sync etc.) + # Two URL patterns are supported, both mapping to nph-tap.py: + # /cgi-bin/TAP/nph-tap.py/ — direct CGI URL + # /TAP/ — short form pyNEID uses, + # mimicking the Apache + # rewrite KOA/NEID/NEA + # run in production. path = parts.path marker = "/nph-tap.py" if marker in path: path_info = path.split(marker, 1)[1] + elif path.startswith("/TAP/"): + path_info = path[len("/TAP"):] # keep leading slash else: path_info = "" diff --git a/tests/test_pyneid_compat.py b/tests/test_pyneid_compat.py new file mode 100644 index 0000000..e927e0a --- /dev/null +++ b/tests/test_pyneid_compat.py @@ -0,0 +1,54 @@ +"""pyNEID compatibility regression tests. + +Currently all skipped — the test fixture's HTTP handler doesn't yet +support TAP's full async flow end-to-end. + +Why these matter: pyNEID's `query_adql` always uses the async TAP +endpoint, which works in three round-trips: + +1. POST to /TAP/async with the ADQL query → server returns 303 See Other + with a Location: header. +2. POST to with PHASE=RUN → server starts the job, returns + another 303 to the same status URL. +3. GET the status URL repeatedly until phase=COMPLETED, then GET the + results URL embedded in the status XML. + +Steps 2–3 depend on the server persisting per-request workspace state +(status.xml, results) under TAP_WORKDIR keyed by a workspace ID +embedded in the URL path. The current `_NphCGIHandler` in +tests/conftest.py routes by URL prefix but doesn't model the workspace +lifecycle — each request is a fresh CGI subprocess that doesn't know +about prior request state. Production runs under Apache, which manages +the same workdir mount across requests so the state naturally +persists, but reproducing that fidelity in CI needs: + +- Sticky working directory across requests (already mostly there since + we set `os.chdir(fixture_root)` once at fixture setup) +- Correct PATH_INFO routing for the second and subsequent calls in the + async flow (the Location URL pyNEID gets back may not match the URL + pattern the handler routes today) +- Possibly more debug — when this was tried locally, pyNEID hung at + step 2 with no observable error in the TAP debug log + +Equivalent direct-HTTP coverage of the same code paths exists in +tests/test_security_e2e.py and tests/test_http_endpoint.py — those +exercise the sync endpoint, which uses a single request and doesn't +need the workspace dance. The pyNEID layer adds value as a regression +check on top of those (it's the exact wrapper production traffic hits), +but is not blocking. + +To wire this up later: trace the async POST/poll loop, fix the handler +routing, then drop the skip decorator. +""" +from __future__ import annotations + +import pytest + +# Once the async fixture support lands, drop this module-level skip and +# re-add the actual test bodies (a snapshot of the intended tests is in +# the git history of this file or in the design plan). +pytest.skip( + "Async TAP flow not yet supported by the test fixture handler. " + "See module docstring for what's needed.", + allow_module_level=True, +) diff --git a/tests/test_security_e2e.py b/tests/test_security_e2e.py new file mode 100644 index 0000000..772ef38 --- /dev/null +++ b/tests/test_security_e2e.py @@ -0,0 +1,252 @@ +"""End-to-end security regression tests. + +Each test is named after the specific regression or security boundary it +guards. The intent is that any future change that re-opens one of these +holes turns the corresponding test red at PR time, making the regression +unmissable in code review. + +Actual TAP status-code behavior in v3.0.0: +- 403 only for `TableValidationError`. The validator raises this for + *system-catalog* references (e.g., sqlite_master, dba_users), and + the table whitelist raises it for tables not in TAP_SCHEMA.tables. +- 400 for everything else, including DML rejection, dangerous-function + rejection, semicolon-chained statements, and ordinary query errors. + This is inconsistent — DML/dangerous-func rejections are security + policy decisions and should arguably also be 403 — but the existing + code raises plain Exception for those, not TableValidationError. + Tests match the actual behavior; harmonizing the codes is a follow-up. +- 500 for startup failures. +""" +from __future__ import annotations + +import pytest +import requests +from _helpers import row_count_from_ipac, tap_sync_url + +# ---------------------------------------------------------------- helpers + +def _post(server: str, query: str, **params) -> requests.Response: + """POST a sync TAP query.""" + data = {"query": query, "format": "ipac"} + data.update(params) + return requests.post(tap_sync_url(server), data=data) + + +def _count(server: str, query: str, **params) -> int: + """Run a query and return the row count from the IPAC response. + + Asserts a 200 before counting — caller should expect success. + """ + r = _post(server, query, **params) + assert r.status_code == 200, f"expected 200 got {r.status_code}: {r.text[:300]}" + return row_count_from_ipac(r.text) + + +# ---------------------------------------------------------------- table whitelist + +def test_table_whitelist_rejects_system_catalog(tap_server: str): + """SQL injection class: system-catalog reads must be blocked. + + Regression marker: cd3d963 (initial table whitelist). + """ + r = _post(tap_server, "select * from sqlite_master") + assert r.status_code == 403, r.text[:300] + + +def test_table_whitelist_rejects_unregistered_table(tap_server: str): + """Tables not in TAP_SCHEMA.tables are 403, not 400 or 200.""" + r = _post(tap_server, "select * from not_in_tap_schema_at_all") + assert r.status_code == 403, r.text[:300] + + +def test_table_whitelist_allows_propfilter_internal_table_in_subquery(tap_server: str): + """The propfilter's access-control table is reachable from subqueries. + + Regression markers: bdbd2d7, de1941f. The fix excluded the configured + ACCESS_TBL / USERS_TBL from validation so the prop check could still + reference them in `WHERE program IN (SELECT program FROM test_access ...)`. + """ + q = ( + "select l1filename from data_l1 " + "where program in (select program from test_access where userid='') " + "or l1propint = 0" + ) + r = _post(tap_server, q, propflag="0") + assert r.status_code == 200, r.text[:300] + + +def test_order_by_not_treated_as_table(tap_server: str): + """ORDER BY columns must not be misidentified as table names. + + Regression marker: 1404e15. sqlparse tokenizes 'ORDER BY' as a single + keyword; the table extractor was treating columns after it as tables + until that fix. + """ + r = _post(tap_server, "select l0filename from data_l0 order by obsdate") + assert r.status_code == 200, r.text[:300] + assert row_count_from_ipac(r.text) == 6 + + +def test_unknown_statement_does_not_skip_validation(tap_server: str): + """Oracle-dialect queries using ROWNUM must still be validated. + + Regression marker: 867278c. sqlparse returns get_type() == 'UNKNOWN' + for ROWNUM-bearing queries; an earlier extractor skipped UNKNOWN + statements entirely, letting them through validation. The fix + extracts tables from UNKNOWN statements too. + """ + r = _post( + tap_server, + "select * from not_in_tap_schema where rownum < 10", + ) + assert r.status_code == 403, r.text[:300] + + +# ---------------------------------------------------------------- propflag + +def test_propflag_zero_honored_for_simple_l0(tap_server: str): + """L0 with propflag=0 must return all rows — the UI's L0 metadata + listing path. data_l0 has 6 rows total (all public, propint=0).""" + n = _count(tap_server, "select l0filename from data_l0", propflag="0") + assert n == 6 + + +@pytest.mark.skip( + reason="Cannot exercise the propflag guard with the SQLite fixture. " + "configparam.py only reads PROPFILTER/ACCESS_TBL/USERS_TBL " + "inside its `if self.dbms == 'oracle'` branch (~lines 255-345), " + "so on a SQLite config those keys are silently dropped and " + "self.config.propfilter ends up '' — neither the NEID nor the " + "KOA guard fires. This is a separate bug worth fixing (the " + "guard semantics should not be Oracle-only). End-to-end propflag " + "coverage will land with the deferred Oracle/Postgres/MySQL CI " + "matrix work." +) +def test_propflag_bypass_routes_l1_through_propfilter(tap_server: str): + """L1 with propflag=0 must route through propfilter, not bare tapQuery. + + Regression markers: PR #16 + PR #17. The guard in tap.py:1612-1638 + forces propflag=1 when datalevel is eng/l1/l2. Cannot be exercised + against the SQLite fixture (see skip reason).""" + # If/when configparam.py is fixed to read PROPFILTER for non-Oracle + # configs, this test will run and assert: + # - propflag=0 status_code != 200 (because propfilter then errors + # on add_months — fixing THAT is a separate concern) + # - propflag=0 status_code == propflag=1 status_code + raise AssertionError("Skipped — see decorator") + + +def test_propflag_whitelist_l0_complex_case_does_not_misfire(tap_server: str): + """The UI's L0 download-link query shape (CASE WHEN with a subquery + FROM nested in the SELECT list) with propflag=0 must succeed and + return rows. + + Regression marker: PR #16 introduced a bug where the propflag guard + used `datalevel != 'l0'`, which evaluates True for the empty + datalevel that __getDatalevel__ returns when the table-name parser + picks the subquery's table first. That bug routed the L0 query + through propfilter, which then constructed malformed prop-check SQL. + + PR #17 switched to an explicit `datalevel in ('eng','l1','l2')` + whitelist so the misfire stopped happening — when datalevel='', the + whitelist doesn't match, and the L0 query stays on the tapQuery path. + + Query structure is the regression trigger: a CASE WHEN containing a + `SELECT ... FROM test_access` subquery in the SELECT list. The + table-name parser yields `test_access` first; __getDatalevel__ on + 'test_access' returns ''. Production used `add_months(obsdate, + l0propint)` in the CASE condition; SQLite doesn't have add_months, + so this test uses a comparison the table-name parser still treats + identically. + """ + q = ( + "select (case when program in " + "(SELECT program from test_access where userid = '') " + "then l0filename else 'redacted' end) as fn, " + "l0filename, program from data_l0" + ) + r = _post(tap_server, q, propflag="0") + assert r.status_code == 200, r.text[:300] + assert row_count_from_ipac(r.text) == 6 + + +# ---------------------------------------------------------------- statement validator + +def test_statement_validator_rejects_dml(tap_server: str): + """INSERT/UPDATE/DELETE/DROP must all be rejected (status 400). + + `validate_statement` raises plain Exception (not TableValidationError) + for these, so the response is 400 rather than 403. Arguably they + should also be 403 — security policy rejection rather than malformed + query — but harmonizing the codes is a separate follow-up. + """ + for dml in ( + "insert into data_l0 (l0filename) values ('x')", + "update data_l0 set l0filename = 'x'", + "delete from data_l0", + "drop table data_l0", + ): + r = _post(tap_server, dml) + assert r.status_code == 400, f"{dml!r} -> {r.status_code}: {r.text[:200]}" + assert "not permitted" in r.text or "rejected" in r.text, r.text[:300] + + +def test_statement_validator_rejects_oracle_dangerous_functions(tap_server: str): + """utl_http, dbms_sql, sys_context etc. must be rejected (status 400). + + Same 400-vs-403 inconsistency as DML rejection — see that test. + """ + for sql in ( + "select utl_http.request('http://evil/x') from data_l0", + "select dbms_sql.parse(1) from data_l0", + "select sys_context('USERENV','IP_ADDRESS') from data_l0", + ): + r = _post(tap_server, sql) + assert r.status_code == 400, f"{sql!r} -> {r.status_code}: {r.text[:200]}" + assert "not permitted" in r.text, r.text[:300] + + +def test_statement_validator_rejects_semicolon_chained(tap_server: str): + """Semicolon-chained statements (classic SQL injection) are rejected. + + Same 400-vs-403 inconsistency as DML rejection. + """ + r = _post( + tap_server, + "select l0filename from data_l0; drop table data_l0", + ) + assert r.status_code == 400, r.text[:300] + assert "semicolon" in r.text.lower(), r.text[:300] + + +# ---------------------------------------------------------------- HTTP status codes + +def test_http_status_code_400_for_query_error(tap_server: str): + """A query referencing a missing column on a valid table → 400.""" + r = _post(tap_server, "select column_that_does_not_exist from data_l0") + # The column doesn't exist; SQLite/TAP returns this as a query error, + # which is 400 (not 403, since the table itself is valid). + assert r.status_code == 400, r.text[:300] + + +# ---------------------------------------------------------------- error message safety + +def test_table_reject_error_message_is_safe(tap_server: str): + """Error responses for rejected tables must not leak internal paths + or expose stack-trace internals. + + Regression marker: 4fa46ff (generic startup error messages) + + c202285 (don't leak raw Oracle errors). This test is a sanity check + that the error response for a table rejection stays clean. + """ + r = _post(tap_server, "select * from sqlite_master") + body = r.text + # No bare file paths (full /tmp/, /var/, /Users/ etc.) + forbidden_substrings = [ + "Traceback", + "Password=", + "/tmp/tap_", + "site-packages", + ] + for s in forbidden_substrings: + assert s not in body, f"Leaked {s!r} in error response:\n{body[:500]}" From 2565a6db75a5b1726735c04693985698fc0dd535 Mon Sep 17 00:00:00 2001 From: BJ Fulton Date: Thu, 10 Sep 2026 15:34:45 -0600 Subject: [PATCH 06/31] Fix malformed HTTP responses from /capabilities and /availability Both VOSI endpoints emitted their XML document with no HTTP status line and no headers at all -- the first bytes on the wire were `/TAP and returned a 500 whenever that directory did not already exist -- so on a fresh deployment these endpoints failed outright, and on an established one they reached the emitter and produced the header-less response above. Add tests/test_vosi_http_response.py, which asserts on raw socket bytes rather than going through a forgiving client library: a CRLF-terminated status line, no bare LF or CR within any header line, CRLFCRLF ending the header block, an XML Content-type, and an intact body. Verified red/green both ways -- the tests fail against the unpatched code and also catch the `HTTP/1.1 200 OK\n` bare-LF variant. Co-Authored-By: Claude Opus 5 --- TAP/tap.py | 47 +++++++++++++---- tests/test_vosi_http_response.py | 89 ++++++++++++++++++++++++++++++++ 2 files changed, 126 insertions(+), 10 deletions(-) create mode 100644 tests/test_vosi_http_response.py diff --git a/TAP/tap.py b/TAP/tap.py index 3442d23..9e6cd41 100755 --- a/TAP/tap.py +++ b/TAP/tap.py @@ -570,6 +570,21 @@ def __run__(self, **kwargs): self.statdict['resulturl'] = '' + # + #{ availability and capabilities are static VOSI documents: they need + # no workspace, so answer them before workspace resolution. Going + # through it treated the empty jobid as a workspace name and failed + # with a 500 whenever /TAP did not already exist. + # + if (self.tapcontext == 'availability'): + self.__printVosiAvailability__ () + + if (self.tapcontext == 'capabilities'): + self.__printVosiCapability__ () + # + #} end VOSI static endpoints + # + # # sync or async without input workspace id: make workspace, # otherwise retrieve workspace from getstatus id @@ -660,16 +675,6 @@ def __run__(self, **kwargs): logging.debug(f'workspace = {self.workspace:s}') logging.debug(f'userWorkdir = {self.userWorkdir:s}') - # - #{ if tapcontext is one of vosiEnpoint, take care of take care of VOSI - # output and return - # - if (self.tapcontext == 'availability'): - self.__printVosiAvailability__ () - - if (self.tapcontext == 'capabilities'): - self.__printVosiCapability__ () - # # vositable: make up vositbl filepath # @@ -3037,6 +3042,17 @@ def __printVosiAvailability__ (self, **kwargs): # { printVosiAvailability # + # + # nph- CGI: emit the full HTTP response ourselves. The status + # line and each header must end in CRLF (print supplies the LF), + # and a bare CRLF line closes the header block -- nginx and + # Cloudflare reject the response otherwise. + # + + print ("HTTP/1.1 200 OK\r") + print ("Content-type: text/xml\r") + print ("\r") + print ('') print ('') print ('') print ('') print (' bytes: + """Issue a GET and return the unparsed response bytes off the socket.""" + parts = urlsplit(server) + with socket.create_connection((parts.hostname, parts.port), timeout=30) as sock: + request = ( + f"GET {path} HTTP/1.1\r\n" + f"Host: {parts.hostname}:{parts.port}\r\n" + "Connection: close\r\n" + "\r\n" + ) + sock.sendall(request.encode("ascii")) + chunks = [] + while True: + chunk = sock.recv(65536) + if not chunk: + break + chunks.append(chunk) + return b"".join(chunks) + + +VOSI_PATHS = ["/TAP/capabilities", "/TAP/availability"] + + +@pytest.mark.parametrize("path", VOSI_PATHS) +def test_vosi_status_line_is_crlf_terminated(tap_server: str, path: str): + """The response opens with a CRLF-terminated 200 status line.""" + raw = _raw_get(tap_server, path) + assert raw.startswith(b"HTTP/1.1 200 OK\r\n"), ( + f"{path} must begin with a CRLF-terminated status line; " + f"got {raw[:60]!r}" + ) + + +@pytest.mark.parametrize("path", VOSI_PATHS) +def test_vosi_headers_are_crlf_terminated(tap_server: str, path: str): + """Every header line uses CRLF, and CRLFCRLF separates headers from body.""" + raw = _raw_get(tap_server, path) + assert b"\r\n\r\n" in raw, f"{path} has no CRLF blank line ending the headers" + + head = raw.split(b"\r\n\r\n", 1)[0] + for line in head.split(b"\r\n"): + assert b"\n" not in line, ( + f"{path} header line {line!r} contains a bare LF" + ) + assert b"\r" not in line, ( + f"{path} header line {line!r} contains a bare CR" + ) + + +@pytest.mark.parametrize("path", VOSI_PATHS) +def test_vosi_declares_xml_content_type(tap_server: str, path: str): + """A Content-type header is present and announces XML.""" + raw = _raw_get(tap_server, path) + head = raw.split(b"\r\n\r\n", 1)[0].lower() + assert b"content-type: text/xml" in head, ( + f"{path} is missing an XML Content-type header; got {head!r}" + ) + + +@pytest.mark.parametrize( + "path,root_element", + [ + ("/TAP/capabilities", b"') + assert root_element in body From 4a12ff92cd8f3026ca25defdc6dadf82b082c540 Mon Sep 17 00:00:00 2001 From: BJ Fulton Date: Thu, 10 Sep 2026 16:05:19 -0600 Subject: [PATCH 07/31] Unblock CI: strip trailing whitespace and pin ruff `ruff check .` has been failing on develop since PR #23 merged (2026-06-02), on a W291 trailing-whitespace hit at TAP/vositables.py:214 introduced by 9024259. The lint step runs before pytest, so the whole test matrix aborts before a single test executes. requirements-test.txt asked for `ruff>=0.1`, so CI silently tracked whatever ruff had shipped most recently; a newer ruff began flagging this line on code nobody had touched. Pin ruff==0.15.2 so rule changes arrive as a deliberate bump rather than a surprise red build. The vositables.py change is whitespace-only -- `git diff -w` is empty. Co-Authored-By: Claude Opus 5 --- TAP/vositables.py | 2 +- requirements-test.txt | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/TAP/vositables.py b/TAP/vositables.py index d414081..2b15001 100644 --- a/TAP/vositables.py +++ b/TAP/vositables.py @@ -211,7 +211,7 @@ def __init__(self, **kwargs): # # { Connect to DBMS # - if('connectInfo' in kwargs): + if('connectInfo' in kwargs): self.connectInfo = kwargs['connectInfo'] else: self.msg = 'Required connectInfo dict is missing.' diff --git a/requirements-test.txt b/requirements-test.txt index 343732f..b4f60d6 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,5 +1,8 @@ pytest>=7 -ruff>=0.1 +# Pinned: ruff's default rule behaviour changes between releases, and an +# unpinned `ruff>=0.1` turned CI red on untouched code (W291 in +# TAP/vositables.py) the moment a new ruff shipped. Bump deliberately. +ruff==0.15.2 requests # Runtime deps. These are not declared in setup.py's install_requires (a From ca7a755e187d71fa36a844f369c6714271556fc4 Mon Sep 17 00:00:00 2001 From: BJ Fulton Date: Thu, 10 Sep 2026 16:33:00 -0600 Subject: [PATCH 08/31] Cover the fresh-deployment case deterministically The existing VOSI tests go through the session-scoped `tap_server` fixture, which shares one TAP_WORKDIR across the whole suite. By the time they run, earlier sync requests have already created /TAP, so a full-suite run only ever exercised the established-server path -- the fresh-deployment case, where the workspace-resolution bug actually returned a 500, was covered only by accident when the file happened to be run in isolation. Add test_vosi_works_without_existing_workspace, which drives the CGI directly against a pristine per-test workdir and asserts both that the response opens with a CRLF-terminated 200 and that the endpoints create no workspace at all. Verified it pins the second defect specifically: reverting only the dispatch move (keeping the header fix) fails these two tests. Co-Authored-By: Claude Opus 5 --- tests/test_vosi_http_response.py | 63 ++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/tests/test_vosi_http_response.py b/tests/test_vosi_http_response.py index 45397e9..de6a5a3 100644 --- a/tests/test_vosi_http_response.py +++ b/tests/test_vosi_http_response.py @@ -9,11 +9,19 @@ """ from __future__ import annotations +import os import socket +import subprocess +import sys +from pathlib import Path from urllib.parse import urlsplit import pytest +REPO_ROOT = Path(__file__).resolve().parent.parent +FIXTURES = Path(__file__).parent / "fixtures" +STUBS = FIXTURES / "stubs" + def _raw_get(server: str, path: str) -> bytes: """Issue a GET and return the unparsed response bytes off the socket.""" @@ -87,3 +95,58 @@ def test_vosi_body_still_intact(tap_server: str, path: str, root_element: bytes) body = raw.split(b"\r\n\r\n", 1)[1] assert body.lstrip().startswith(b'') assert root_element in body + + +def _run_cgi_with_workdir(tmp_path, fixture_root, path_info: str) -> bytes: + """Run the CGI directly against a *pristine* TAP_WORKDIR. + + The session `tap_server` fixture shares one workdir across the whole + suite, so by the time these tests run `/TAP` has already been + created by earlier sync requests. That hides the fresh-deployment case, + which is exactly where the workspace-resolution bug bit. Point the CGI at + an empty workdir so the no-workspace path is exercised deterministically. + """ + workdir = tmp_path / "workdir" + workdir.mkdir() + conf = tmp_path / "TAP.conf" + conf.write_text( + (FIXTURES / "TAP.conf.template").read_text().format( + TEST_WORKDIR=str(workdir), + TEST_HTTP_URL="http://127.0.0.1:8099", + TEST_DB_PATH=str(fixture_root / "test_data.db"), + TEST_TAP_SCHEMA=str(fixture_root / "tap_schema.db"), + ) + ) + + env = os.environ.copy() + env.update({ + "TAP_CONF": str(conf), + "PATH_INFO": path_info, + "REQUEST_METHOD": "GET", + "QUERY_STRING": "", + "PYTHONPATH": os.pathsep.join([str(REPO_ROOT), str(STUBS)]), + }) + proc = subprocess.run( + [sys.executable, str(fixture_root / "cgi-bin" / "TAP" / "nph-tap.py")], + env=env, capture_output=True, timeout=60, + ) + assert (workdir / "TAP").exists() is False, ( + "the VOSI endpoints must not create a workspace" + ) + return proc.stdout + + +@pytest.mark.parametrize("path", VOSI_PATHS) +def test_vosi_works_without_existing_workspace(tmp_path, fixture_root, path): + """A fresh deployment, where /TAP does not yet exist, still works. + + These endpoints previously fell through to the "retrieve workspace from + jobid" branch with an empty jobid, resolving to /TAP and + returning a 500 whenever that directory was absent. + """ + raw = _run_cgi_with_workdir(tmp_path, fixture_root, path.replace("/TAP", "", 1)) + assert raw.startswith(b"HTTP/1.1 200 OK\r\n"), ( + f"{path} on a pristine workdir must return a CRLF-terminated 200; " + f"got {raw[:120]!r}" + ) + assert b"500" not in raw.split(b"\r\n", 1)[0] From a0955e34a741535ac1a7a25d14e60ff332044922 Mon Sep 17 00:00:00 2001 From: BJ Fulton Date: Thu, 10 Sep 2026 17:06:12 -0600 Subject: [PATCH 09/31] Spell out the CRLF terminator with end='\r\n' Review preference from @jpl-jengelke on #26, mirrored here to keep the two branches in sync. Make the CRLF requirement explicit at each print rather than relying on a trailing \r plus print's implicit \n. Byte output is unchanged -- both forms emit `...\r\n`, and the wire-level tests in tests/test_vosi_http_response.py still pass unmodified -- but the intent is legible without having to remember what print appends, which matters on a file where a bare LF is what broke the endpoint. Co-Authored-By: Claude Opus 5 --- TAP/tap.py | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/TAP/tap.py b/TAP/tap.py index 9e6cd41..05791de 100755 --- a/TAP/tap.py +++ b/TAP/tap.py @@ -3044,14 +3044,16 @@ def __printVosiAvailability__ (self, **kwargs): # # nph- CGI: emit the full HTTP response ourselves. The status - # line and each header must end in CRLF (print supplies the LF), - # and a bare CRLF line closes the header block -- nginx and - # Cloudflare reject the response otherwise. + # line and each header must end in CRLF, and a bare CRLF line + # closes the header block -- nginx and Cloudflare reject the + # response otherwise. The terminator is spelled out via end= + # rather than a trailing \r leaning on print's implicit \n, + # so the CRLF requirement is visible at a glance. # - print ("HTTP/1.1 200 OK\r") - print ("Content-type: text/xml\r") - print ("\r") + print ('HTTP/1.1 200 OK', end='\r\n') + print ('Content-type: text/xml', end='\r\n') + print ('', end='\r\n') print ('') print ('') @@ -3078,14 +3080,16 @@ def __printVosiCapability__ (self, **kwargs): # # nph- CGI: emit the full HTTP response ourselves. The status - # line and each header must end in CRLF (print supplies the LF), - # and a bare CRLF line closes the header block -- nginx and - # Cloudflare reject the response otherwise. + # line and each header must end in CRLF, and a bare CRLF line + # closes the header block -- nginx and Cloudflare reject the + # response otherwise. The terminator is spelled out via end= + # rather than a trailing \r leaning on print's implicit \n, + # so the CRLF requirement is visible at a glance. # - print ("HTTP/1.1 200 OK\r") - print ("Content-type: text/xml\r") - print ("\r") + print ('HTTP/1.1 200 OK', end='\r\n') + print ('Content-type: text/xml', end='\r\n') + print ('', end='\r\n') print ('') print ('') From 202f4d127d42b95307ade28ff435744323d0ade5 Mon Sep 17 00:00:00 2001 From: BJ Fulton Date: Fri, 11 Sep 2026 13:23:08 -0600 Subject: [PATCH 10/31] Serve the VOSI documents as application/xml The NEA deployment already answers /capabilities and /availability with application/xml, and that is the type we want for these documents. Match it here so the two do not diverge, and update the wire-level assertion to match. Scope is deliberately the two VOSI emitters this PR already touches; the other Content-type sites (including __printVosiTables__ and the error paths) are left alone. Co-Authored-By: Claude Opus 5 --- TAP/tap.py | 4 ++-- tests/test_vosi_http_response.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/TAP/tap.py b/TAP/tap.py index 05791de..89d6e2f 100755 --- a/TAP/tap.py +++ b/TAP/tap.py @@ -3052,7 +3052,7 @@ def __printVosiAvailability__ (self, **kwargs): # print ('HTTP/1.1 200 OK', end='\r\n') - print ('Content-type: text/xml', end='\r\n') + print ('Content-type: application/xml', end='\r\n') print ('', end='\r\n') print ('') @@ -3088,7 +3088,7 @@ def __printVosiCapability__ (self, **kwargs): # print ('HTTP/1.1 200 OK', end='\r\n') - print ('Content-type: text/xml', end='\r\n') + print ('Content-type: application/xml', end='\r\n') print ('', end='\r\n') print ('') diff --git a/tests/test_vosi_http_response.py b/tests/test_vosi_http_response.py index de6a5a3..860fb75 100644 --- a/tests/test_vosi_http_response.py +++ b/tests/test_vosi_http_response.py @@ -77,7 +77,7 @@ def test_vosi_declares_xml_content_type(tap_server: str, path: str): """A Content-type header is present and announces XML.""" raw = _raw_get(tap_server, path) head = raw.split(b"\r\n\r\n", 1)[0].lower() - assert b"content-type: text/xml" in head, ( + assert b"content-type: application/xml" in head, ( f"{path} is missing an XML Content-type header; got {head!r}" ) From 319effc8a290e769d1ece4a2326212d2d38ff40a Mon Sep 17 00:00:00 2001 From: Webserver Operations Date: Mon, 21 Sep 2026 20:18:01 -0600 Subject: [PATCH 11/31] test: make the fixture able to host an async job Three things stood between the test fixture and the async endpoint, none of them related to each other: - The composed job URLs were malformed. configparam.py appends HTTP_PORT to HTTP_URL for any port other than 80/443, and the fixture had the port in both, so the status URL came back as host:port:0. No sync test reads httpurl, which is why this never surfaced. Split the template into TEST_HTTP_HOST and TEST_HTTP_PORT. - A CGI that dies has only stderr to say so, and the fixture handler dropped it, which makes a 500 from the script indistinguishable from a 500 the script meant to send. Surface it on the runner's stderr. - tap.py asks BeautifulSoup for the 'lxml' tree builder by name when it reads a status document (TAP/tap.py:904, 2162), so bs4 alone is not enough for anything that touches an existing job. Add lxml to the test requirements. (setup.py's install_requires is missing it too, along with bs4, xmltodict, astropy and sqlparse; that is a packaging fix for its own PR.) Also strips one trailing space in TAP/vositables.py that came in with the v3.0.1 merge and is currently failing `ruff check .` on develop. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01Q5cRrMXoPJ6zSiGfKYcqjC --- TAP/vositables.py | 2 +- requirements-test.txt | 4 ++++ tests/conftest.py | 9 ++++++++- tests/fixtures/TAP.conf.template | 11 ++++++++--- 4 files changed, 21 insertions(+), 5 deletions(-) diff --git a/TAP/vositables.py b/TAP/vositables.py index d414081..2b15001 100644 --- a/TAP/vositables.py +++ b/TAP/vositables.py @@ -211,7 +211,7 @@ def __init__(self, **kwargs): # # { Connect to DBMS # - if('connectInfo' in kwargs): + if('connectInfo' in kwargs): self.connectInfo = kwargs['connectInfo'] else: self.msg = 'Required connectInfo dict is missing.' diff --git a/requirements-test.txt b/requirements-test.txt index 343732f..66043b2 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -10,6 +10,10 @@ configobj sqlparse astropy beautifulsoup4 +# tap.py asks BeautifulSoup for the 'lxml' tree builder by name when it +# reads a job status document (TAP/tap.py:904, 2162), so bs4 alone is not +# enough for the async flow. +lxml xmltodict # pyneid is not currently used — tests/test_pyneid_compat.py is skipped diff --git a/tests/conftest.py b/tests/conftest.py index 6aa0cf6..e0ab97c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -66,7 +66,8 @@ def tap_conf(fixture_root: Path) -> Path: template = (FIXTURES / "TAP.conf.template").read_text() conf = template.format( TEST_WORKDIR=str(fixture_root / "workdir"), - TEST_HTTP_URL=f"http://127.0.0.1:{port}", + TEST_HTTP_HOST="127.0.0.1", + TEST_HTTP_PORT=str(port), TEST_DB_PATH=str(fixture_root / "test_data.db"), TEST_TAP_SCHEMA=str(fixture_root / "tap_schema.db"), ) @@ -147,6 +148,12 @@ def _run_cgi(self): ) stdout, stderr = proc.communicate(input=body) + # A CGI that dies has only stderr to say so, and swallowing it + # makes a 500 from the script indistinguishable from a 500 the + # script meant to send. Surface it on the test runner's stderr. + if stderr: + sys.stderr.write(stderr.decode("utf-8", "replace")) + # Forward raw subprocess stdout to the socket. The script is # responsible for emitting a valid HTTP status line + headers. try: diff --git a/tests/fixtures/TAP.conf.template b/tests/fixtures/TAP.conf.template index e0280d2..6c2c4a9 100644 --- a/tests/fixtures/TAP.conf.template +++ b/tests/fixtures/TAP.conf.template @@ -5,7 +5,8 @@ # # Placeholders: # {TEST_WORKDIR} — working dir for TAP request artifacts -# {TEST_HTTP_URL} — http://localhost:NNNN (fixture-allocated) +# {TEST_HTTP_HOST} — 127.0.0.1 +# {TEST_HTTP_PORT} — fixture-allocated port # {TEST_DB_PATH} — absolute path to test_data.db # {TEST_TAP_SCHEMA} — absolute path to tap_schema.db @@ -13,8 +14,12 @@ TAP_WORKDIR={TEST_WORKDIR} TAP_WORKURL=/workspace - HTTP_URL={TEST_HTTP_URL} - HTTP_PORT=0 + # configparam appends HTTP_PORT to HTTP_URL for any port other than + # 80/443, and that composed value is what the async status and result + # URLs are built from. Keep the two apart here or the job URLs come + # back with the port twice. + HTTP_URL=http://{TEST_HTTP_HOST} + HTTP_PORT={TEST_HTTP_PORT} CGI_PGM=TAP From a59eb90cea979f8dec5732923a511019cd9a9673 Mon Sep 17 00:00:00 2001 From: Webserver Operations Date: Mon, 21 Sep 2026 20:18:19 -0600 Subject: [PATCH 12/31] fix: detach async jobs by forking, not by killing the web server An async submit has to finish an HTTP response now and keep executing the query afterwards. Under CGI those pull against each other: the server completes the response when the script closes stdout, and mod_cgi terminates the script once the request is cleaned up. So the process that runs the query can be neither the one holding stdout nor a member of the request's process group. __printAsyncResponse__ resolved that by sending the 303, sleeping two seconds, and then SIGKILLing os.getppid(). Under CGI that parent is the web server child serving the request. Killing it does end the response, since the body carried no Content-Length, and it does orphan the CGI so the job survives. But the client's connection dies mid-message, and behind a reverse proxy the proxy is the one holding that connection: it marks the backend in error and answers 5xx for the next request or two routed over it. Bare mod_cgi respawned the killed child silently, so the damage only became visible once a proxy was put in front. Fork instead. The parent records the child as the job's runId, writes the status document, sends the 303 and exits, which ends the response the way any other CGI does. The child calls setsid() to leave the request's process group, points its standard streams at /dev/null, and runs the query. Three details are load-bearing: - The child waits on a pipe until the parent has published the status document. Without that, a fast query can write COMPLETED before the parent writes EXECUTING, and the job looks like it regressed. - The child redirects stdio before it waits, so it never holds a dup of the server pipe (which would delay the response) and can never write into a response that has already been sent. - uws:runId now carries the child's pid, because ABORT kills whatever runId names (TAP/tap.py:1125-1136). A stale pid there would either do nothing or, worse, signal an unrelated process. The response is also self-delimiting now: an nph- script gets no Content-Length from the server, and a proxy has no other way to know where the message ends. If fork() fails, the request is answered from this process and the query runs here, which is the old behavior minus the kill: correct response, job tied to the request's lifetime. tests/test_async_flow.py covers the full three round trips (submit, PHASE=RUN, poll to COMPLETED plus a non-empty result on disk), asserts the 303 is framed, and guards the regression at the source. That last one is a source assertion on purpose: the behavioral version is "the process running this suite is still alive", which a suite that has been SIGKILLed cannot make. Before this change, running these tests printed "Killed" and nothing else. tests/test_pyneid_compat.py recorded that pyNEID "hung at step 2 with no observable error in the TAP debug log". Step 2 is PHASE=RUN. Its docstring is corrected to say what the hang actually was. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01Q5cRrMXoPJ6zSiGfKYcqjC --- TAP/tap.py | 188 ++++++++++++++++++++++++++++++++---- tests/test_async_flow.py | 157 ++++++++++++++++++++++++++++++ tests/test_pyneid_compat.py | 66 +++++-------- 3 files changed, 354 insertions(+), 57 deletions(-) create mode 100644 tests/test_async_flow.py diff --git a/TAP/tap.py b/TAP/tap.py index 3442d23..a901852 100755 --- a/TAP/tap.py +++ b/TAP/tap.py @@ -1286,18 +1286,11 @@ def __run__(self, **kwargs): logging.debug (f'statuspath= {self.statuspath:s}') - self.__writeStatusMsg__(self.statuspath, self.statdict, - self.param) + self.__respondAsyncAndDetach__() if self.debug: logging.debug('') - logging.debug ('call printAsyncResponse') - - self.__printAsyncResponse__(self.statusurl) - - if self.debug: - logging.debug('') - logging.debug('returned printAsyncResponse') + logging.debug('async response sent; running job detached') # # } end async submit case @@ -2726,25 +2719,186 @@ def __printSyncResponse__(self, status, msg, resulturl, format, **kwargs): def __printAsyncResponse__(self, statusurl, **kwargs): # - # async: return statusurl and kill the parent process + # async: point the client at the job's status URL. + # + # The body is one line, but it still needs a Content-Length. This + # is an nph- script, so nothing downstream supplies one, and a + # reverse proxy in front of the CGI has no other way to tell + # where the response ends. Earlier versions omitted it and let + # the connection dying stand in for the end of the message. # - print("HTTP/1.1 303 See Other\r") - print("Location: %s\r\n\r" % statusurl) - print("Redirect Location: %s" % statusurl) + body = 'Redirect Location: %s\n' % statusurl + + sys.stdout.write('HTTP/1.1 303 See Other\r\n') + sys.stdout.write('Location: %s\r\n' % statusurl) + sys.stdout.write('Content-Type: text/plain\r\n') + sys.stdout.write('Content-Length: %d\r\n' + % len(body.encode('utf-8'))) + sys.stdout.write('Connection: close\r\n') + sys.stdout.write('\r\n') + sys.stdout.write(body) sys.stdout.flush() - time.sleep(2.0) + if self.debug: + logging.debug('') + logging.debug(f'async response sent: statusurl= {statusurl:s}') + + return + + + def __respondAsyncAndDetach__(self, **kwargs): + + # + # { An async submit has to finish an HTTP response now and keep + # executing the query afterwards, and under CGI those two pull + # against each other: the web server completes the response + # when the script closes stdout, and mod_cgi terminates the + # script once the request is cleaned up. The process that runs + # the query can therefore be neither the one holding stdout nor + # part of the request. + # + # So fork. The parent names the child as the job's runId, + # writes the status document, sends the 303 and exits, which + # ends the response the way any other CGI would. The child + # leaves the request's process group, points its standard + # streams away from the server pipe, waits for the parent to + # confirm the job is published, and returns to run the query. + # + # Earlier versions sent the response and then SIGKILLed + # os.getppid(). Under CGI that parent is the web server child + # serving the request: killing it truncates the response, and + # behind a reverse proxy it leaves the proxy holding a dead + # upstream connection, which the proxy reports as 503 on the + # next request or two routed over it. + # + + try: + readfd, writefd = os.pipe() + pid = os.fork() + + except OSError as e: + + # + # No fork available: answer the request from this process and + # run the job here. The response is still well formed; the + # job now lives and dies with the request. + # + + logging.error(f'Could not fork async worker: {str(e)}') + + self.__writeStatusMsg__(self.statuspath, self.statdict, + self.param) + + self.__printAsyncResponse__(self.statusurl) + + return + + if (pid > 0): + # + # { parent: publish the job, answer the client, exit + # + os.close(readfd) + + self.statdict['process_id'] = pid + + self.__writeStatusMsg__(self.statuspath, self.statdict, + self.param) + + self.__printAsyncResponse__(self.statusurl) + + # + # The child blocks until this byte arrives, so a fast query + # cannot overwrite the status document written just above. + # + + try: + os.write(writefd, b'1') + + except OSError as e: + logging.error(f'Could not release async worker: {str(e)}') + os.close(writefd) + + sys.exit(0) + # + # } end parent # - # Shut down parent program + # + # { child: detach from the request, then run the query + # + os.close(writefd) + + self.pid = os.getpid() + self.statdict['process_id'] = self.pid + + self.__detachFromServer__() + + published = b'' + + try: + published = os.read(readfd, 1) + + except OSError as e: + logging.error(f'Async worker handshake failed: {str(e)}') + + os.close(readfd) + + if (len(published) == 0): + + # + # The parent exited before it published the job, so there is + # no status document for this run to update. + # - os.kill(os.getppid(), signal.SIGKILL) + logging.error('Async parent exited before publishing the job') + + os._exit(1) if self.debug: logging.debug('') - logging.debug('parent process killed') + logging.debug(f'async worker detached: pid= {self.pid:d}') + + return + # + # } end child + # + + + def __detachFromServer__(self, **kwargs): + + # + # Give up the web server's request context: leave the request's + # process group so the server cannot reap this process along with + # the request, and replace the inherited stdio with /dev/null so + # the response the parent just sent is neither held open by nor + # corrupted from here. + # + + try: + os.setsid() + + except OSError as e: + logging.error(f'setsid failed in async worker: {str(e)}') + + try: + sys.stdout.flush() + sys.stderr.flush() + + except (OSError, ValueError) as e: + logging.error(f'Could not flush async worker stdio: {str(e)}') + + devnull = os.open(os.devnull, os.O_RDWR) + + try: + os.dup2(devnull, 0) + os.dup2(devnull, 1) + os.dup2(devnull, 2) + + finally: + if (devnull > 2): + os.close(devnull) return diff --git a/tests/test_async_flow.py b/tests/test_async_flow.py new file mode 100644 index 0000000..e7e7b3c --- /dev/null +++ b/tests/test_async_flow.py @@ -0,0 +1,157 @@ +"""End-to-end coverage of the UWS async flow. + +An async TAP job takes three round trips, the same sequence pyvo and +pyNEID drive: + +1. POST to ``/async`` with the query. The server creates the job in + PENDING and answers 303 with ``Location: ``. +2. POST ``PHASE=RUN`` to ``/phase``. The server starts the + job and answers 303 to the same status URL. +3. GET ```` until the phase reaches COMPLETED, then read the + result named in the status document. + +None of this could be covered before the detach fix. ``tap.py`` ended +every async response by SIGKILLing ``os.getppid()`` — under this fixture +that is the process running the test suite, and in production it is the +web server child serving the request. Step 2 therefore either hung (the +CGI never closed the stdout pipe the server was reading) or took the +test runner down with it. +""" +from __future__ import annotations + +import time +import xml.etree.ElementTree as ET +from pathlib import Path + +import requests +from _helpers import tap_async_url + +UWS = {"uws": "http://www.ivoa.net/xml/UWS/v1.0", + "xlink": "http://www.w3.org/1999/xlink"} + +QUERY = "select * from data_l0" + + +def _submit(server: str) -> str: + """Round trip 1. Returns the job's status URL.""" + resp = requests.post( + tap_async_url(server), + data={"query": QUERY, "format": "ipac"}, + allow_redirects=False, + timeout=30, + ) + assert resp.status_code == 303, resp.text[:500] + statusurl = resp.headers["Location"] + assert statusurl, "303 carried no Location header" + return statusurl + + +def _status(statusurl: str) -> ET.Element: + resp = requests.get(statusurl, timeout=30) + assert resp.status_code == 200, resp.text[:500] + return ET.fromstring(resp.text) + + +def _phase(statusurl: str) -> str: + return _status(statusurl).find("uws:phase", UWS).text + + +def test_async_submit_creates_pending_job(tap_server): + statusurl = _submit(tap_server) + assert _phase(statusurl) == "PENDING" + + +def test_async_run_starts_the_job(tap_server): + """Round trip 2 answers promptly and does not take the server with it.""" + statusurl = _submit(tap_server) + + resp = requests.post( + f"{statusurl}/phase", + data={"PHASE": "RUN"}, + allow_redirects=False, + timeout=30, + ) + + assert resp.status_code == 303, resp.text[:500] + assert resp.headers["Location"] == statusurl + + +def test_async_run_response_is_self_delimiting(tap_server): + """The 303 must be framed by its headers, not by the connection dying. + + A reverse proxy in front of the CGI needs a Content-Length (or a + chunked body) to relay the response. The old code emitted a body + with neither and relied on the connection being destroyed to mark + the end of the message. + """ + statusurl = _submit(tap_server) + + resp = requests.post( + f"{statusurl}/phase", + data={"PHASE": "RUN"}, + allow_redirects=False, + timeout=30, + ) + + assert "Content-Length" in resp.headers + assert int(resp.headers["Content-Length"]) == len(resp.content) + + +def test_async_job_runs_to_completion(tap_server, fixture_root: Path): + statusurl = _submit(tap_server) + requests.post( + f"{statusurl}/phase", + data={"PHASE": "RUN"}, + allow_redirects=False, + timeout=30, + ) + + deadline = time.time() + 30 + phase = "" + while time.time() < deadline: + phase = _phase(statusurl) + if phase in ("COMPLETED", "ERROR", "ABORTED"): + break + time.sleep(0.2) + + assert phase == "COMPLETED", f"job ended in phase {phase}" + + job = _status(statusurl) + href = job.find("uws:results/uws:result", UWS).get( + "{http://www.w3.org/1999/xlink}href") + assert href + + # The fixture server does not serve TAP_WORKURL (Apache does that in + # production), so read the result out of the workspace on disk. + # tap.py puts each job under /TAP/. + workspace = job.find("uws:jobId", UWS).text + results = list((fixture_root / "workdir" / "TAP" / workspace).glob("*")) + assert results, f"no result artifacts in workspace {workspace}" + written = [p for p in results if p.name != "status.xml" and p.stat().st_size] + assert written, f"job completed but wrote no result file: {results}" + + +def test_async_detach_does_not_signal_the_web_server(): + """Regression guard against the SIGKILL-the-parent detach. + + Asserted on the source rather than on behavior on purpose: the + behavioral assertion is "the process running this suite is still + alive", which a suite that has been SIGKILLed cannot make. + """ + import tokenize + + from TAP import tap + + # Comments and strings are stripped before the check so the code can + # still explain in prose why it no longer signals its parent. + with open(tap.__file__, "rb") as fp: + src = "".join( + tok.string + for tok in tokenize.tokenize(fp.readline) + if tok.type not in (tokenize.COMMENT, tokenize.STRING)) + + assert "getppid" not in src, ( + "tap.py signals its parent process. Under CGI the parent is the " + "web server child serving the request; killing it drops the " + "response and, behind a proxy, poisons the upstream connection." + ) diff --git a/tests/test_pyneid_compat.py b/tests/test_pyneid_compat.py index e927e0a..883a3b7 100644 --- a/tests/test_pyneid_compat.py +++ b/tests/test_pyneid_compat.py @@ -1,54 +1,40 @@ """pyNEID compatibility regression tests. -Currently all skipped — the test fixture's HTTP handler doesn't yet -support TAP's full async flow end-to-end. +Still skipped, but for a different reason than before. -Why these matter: pyNEID's `query_adql` always uses the async TAP -endpoint, which works in three round-trips: +pyNEID's ``query_adql`` always uses the async TAP endpoint, which works +in three round trips: -1. POST to /TAP/async with the ADQL query → server returns 303 See Other +1. POST to /TAP/async with the ADQL query -> server returns 303 See Other with a Location: header. -2. POST to with PHASE=RUN → server starts the job, returns - another 303 to the same status URL. +2. POST to /phase with PHASE=RUN -> server starts the job and + returns another 303 to the same status URL. 3. GET the status URL repeatedly until phase=COMPLETED, then GET the - results URL embedded in the status XML. - -Steps 2–3 depend on the server persisting per-request workspace state -(status.xml, results) under TAP_WORKDIR keyed by a workspace ID -embedded in the URL path. The current `_NphCGIHandler` in -tests/conftest.py routes by URL prefix but doesn't model the workspace -lifecycle — each request is a fresh CGI subprocess that doesn't know -about prior request state. Production runs under Apache, which manages -the same workdir mount across requests so the state naturally -persists, but reproducing that fidelity in CI needs: - -- Sticky working directory across requests (already mostly there since - we set `os.chdir(fixture_root)` once at fixture setup) -- Correct PATH_INFO routing for the second and subsequent calls in the - async flow (the Location URL pyNEID gets back may not match the URL - pattern the handler routes today) -- Possibly more debug — when this was tried locally, pyNEID hung at - step 2 with no observable error in the TAP debug log - -Equivalent direct-HTTP coverage of the same code paths exists in -tests/test_security_e2e.py and tests/test_http_endpoint.py — those -exercise the sync endpoint, which uses a single request and doesn't -need the workspace dance. The pyNEID layer adds value as a regression -check on top of those (it's the exact wrapper production traffic hits), -but is not blocking. - -To wire this up later: trace the async POST/poll loop, fix the handler -routing, then drop the skip decorator. + result URL embedded in the status XML. + +This module used to say the fixture handler could not carry that +sequence, and recorded that "pyNEID hung at step 2 with no observable +error in the TAP debug log". The hang was not the fixture. Step 2 ended +by SIGKILLing os.getppid(), which under the fixture is the test runner +and in production is the web server child serving the request, so the +client was left waiting on a pipe nobody would close. That is fixed, and +the sequence above now has direct coverage in tests/test_async_flow.py. + +What is still missing here is pyNEID itself: it is not a test dependency +and is not installed in CI, so there is nothing to import. Adding it +means weighing a heavier test dependency (and its own network defaults) +against coverage that tests/test_async_flow.py already provides at the +HTTP level. Until that call is made, this module stays skipped. + +To wire it up later: add pyNEID to requirements-test.txt, write the +round-trip test against the fixture server, and drop the skip below. """ from __future__ import annotations import pytest -# Once the async fixture support lands, drop this module-level skip and -# re-add the actual test bodies (a snapshot of the intended tests is in -# the git history of this file or in the design plan). pytest.skip( - "Async TAP flow not yet supported by the test fixture handler. " - "See module docstring for what's needed.", + "pyNEID is not a test dependency; the async flow it drives is covered " + "directly in tests/test_async_flow.py. See module docstring.", allow_module_level=True, ) From 66633460c7d8b4c50e2f55fe2de7ad4354548138 Mon Sep 17 00:00:00 2001 From: Webserver Operations Date: Mon, 21 Sep 2026 20:34:56 -0600 Subject: [PATCH 13/31] fix: stop async ABORT and bad phases from running the job Both from Copilot's review, both real. The async block ends in one shared response path, and every phase reaches it. ABORT and unrecognized phases decide their terminal status, hit that path, and then fall through into the query, which overwrites ABORTED (or ERROR) with COMPLETED. So a client that aborted a PENDING job was told ABORTED and got its results anyway. That predates this branch: the old code wrote the status, sent the 303, killed the server and then ran the query in the orphan. The fork made it easier to see, not worse. The dead per-branch response calls still visible in the RUN and ABORT branches show the original intent, hoisting them into a shared tail is what dropped the exit. Terminal phases now write their status, answer the client, and sys.exit(), which is what __writeAsyncError__ and __printError__ already do everywhere else in this file. Only RUN detaches. Second: the parent released the child after sending the response, so a client or proxy that hung up mid-write took the exception path, skipped the release, and the child saw EOF and exited, leaving a published job stuck in EXECUTING forever. The release moves into a finally and is gated on the status write having succeeded: a hung-up client does not make the job go away, but if there is no job document there is nothing for the child to update. Tests cover both terminal phases, including that nothing runs afterwards (phase holds and no result file appears). Both fail against the previous commit. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01Q5cRrMXoPJ6zSiGfKYcqjC --- TAP/tap.py | 92 +++++++++++++++++++++++++++++++--------- tests/test_async_flow.py | 55 ++++++++++++++++++++++++ 2 files changed, 127 insertions(+), 20 deletions(-) diff --git a/TAP/tap.py b/TAP/tap.py index a901852..61213c1 100755 --- a/TAP/tap.py +++ b/TAP/tap.py @@ -1280,17 +1280,42 @@ def __run__(self, **kwargs): # } end async bogus value case # - if self.debug: - logging.debug('') - logging.debug ('call writeStatusMsg') - logging.debug (f'statuspath= {self.statuspath:s}') + if (self.param['phase'] == 'RUN'): + # + # Publish the job, answer the client, and keep running + # the query in a detached child. + # - self.__respondAsyncAndDetach__() + self.__respondAsyncAndDetach__() - if self.debug: - logging.debug('') - logging.debug('async response sent; running job detached') + if self.debug: + logging.debug('') + logging.debug('async response sent; job running detached') + + else: + + # + # ABORT and unrecognized phases are terminal: the branch + # above has already decided the phase, so write it, + # answer the client and stop. Falling through to the + # query path below would run the job the client just + # aborted and overwrite ABORTED (or ERROR) with + # COMPLETED. + # + + if self.debug: + logging.debug('') + logging.debug ('terminal async phase= ' + f"{self.statdict['phase']:s}") + logging.debug (f'statuspath= {self.statuspath:s}') + + self.__writeStatusMsg__(self.statuspath, self.statdict, + self.param) + + self.__printAsyncResponse__(self.statusurl) + + sys.exit() # # } end async submit case @@ -2802,23 +2827,50 @@ def __respondAsyncAndDetach__(self, **kwargs): self.statdict['process_id'] = pid - self.__writeStatusMsg__(self.statuspath, self.statdict, - self.param) + published = False - self.__printAsyncResponse__(self.statusurl) + try: + self.__writeStatusMsg__(self.statuspath, self.statdict, + self.param) - # - # The child blocks until this byte arrives, so a fast query - # cannot overwrite the status document written just above. - # + published = True - try: - os.write(writefd, b'1') + self.__printAsyncResponse__(self.statusurl) + + except Exception as e: + + # + # Most likely the client or the proxy hung up while the + # response was going out. Nothing can be sent about it + # now; the job's own fate is decided in the finally + # clause below. + # + + logging.error(f'Async response failed: {str(e)}') + + finally: + + # + # The child blocks on this byte, so a fast query cannot + # overwrite the status document written above. Once the + # job is published the child has to be released even if + # the response itself failed: a client that hung up does + # not make the job go away, and a child that exits here + # would leave the job stuck in EXECUTING forever. If the + # status write is what failed, the byte is withheld and + # the child exits, because there is no job document for + # it to update. + # + + if published: + try: + os.write(writefd, b'1') - except OSError as e: - logging.error(f'Could not release async worker: {str(e)}') + except OSError as e: + logging.error( + f'Could not release async worker: {str(e)}') - os.close(writefd) + os.close(writefd) sys.exit(0) # diff --git a/tests/test_async_flow.py b/tests/test_async_flow.py index e7e7b3c..b8e2201 100644 --- a/tests/test_async_flow.py +++ b/tests/test_async_flow.py @@ -131,6 +131,61 @@ def test_async_job_runs_to_completion(tap_server, fixture_root: Path): assert written, f"job completed but wrote no result file: {results}" +def _post_phase(statusurl: str, phase: str): + return requests.post( + f"{statusurl}/phase", + data={"PHASE": phase}, + allow_redirects=False, + timeout=30, + ) + + +def _results(fixture_root: Path, workspace: str) -> list: + workdir = fixture_root / "workdir" / "TAP" / workspace + return [p for p in workdir.glob("*") if p.name != "status.xml"] + + +def test_async_abort_is_terminal(tap_server, fixture_root: Path): + """ABORT must end the job, not run it. + + Every phase shares one response path at the end of the async block, + and that path used to fall through into the query. So a POST of + PHASE=ABORT wrote ABORTED, answered the client, and then ran the + job anyway, overwriting the phase the client had just been told. + """ + statusurl = _submit(tap_server) + assert _phase(statusurl) == "PENDING" + + resp = _post_phase(statusurl, "ABORT") + assert resp.status_code == 303, resp.text[:500] + + job = _status(statusurl) + assert job.find("uws:phase", UWS).text == "ABORTED" + + workspace = job.find("uws:jobId", UWS).text + time.sleep(1.0) + assert _phase(statusurl) == "ABORTED", "the aborted job kept running" + assert not _results(fixture_root, workspace), \ + "the aborted job wrote a result" + + +def test_async_unknown_phase_is_terminal(tap_server, fixture_root: Path): + """An unrecognized phase reports ERROR and runs nothing.""" + statusurl = _submit(tap_server) + + resp = _post_phase(statusurl, "SPIN") + assert resp.status_code == 303, resp.text[:500] + + job = _status(statusurl) + assert job.find("uws:phase", UWS).text == "ERROR" + + workspace = job.find("uws:jobId", UWS).text + time.sleep(1.0) + assert _phase(statusurl) == "ERROR", "the rejected job kept running" + assert not _results(fixture_root, workspace), \ + "the rejected job wrote a result" + + def test_async_detach_does_not_signal_the_web_server(): """Regression guard against the SIGKILL-the-parent detach. From e1656ca1a139cfad7170317ddcafce446a09a6b6 Mon Sep 17 00:00:00 2001 From: BJ Fulton Date: Tue, 6 Oct 2026 10:35:41 -0600 Subject: [PATCH 14/31] Use tap_schema_file only for SQLite configparam always passes tap_schema_file='TAP_SCHEMA.db', SQLite's ATTACH name. Since 4b35392 TableValidator and dataDictionary preferred it for every DBMS, so Oracle got 'FROM TAP_SCHEMA.db.tables' (ORA-03048) on every query. Co-Authored-By: Claude Haiku 4.5 --- TAP/datadictionary.py | 3 +- TAP/tablevalidator.py | 2 +- tests/test_tap_schema_name.py | 68 +++++++++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+), 2 deletions(-) create mode 100644 tests/test_tap_schema_name.py diff --git a/TAP/datadictionary.py b/TAP/datadictionary.py index 7d27719..a11cc2d 100644 --- a/TAP/datadictionary.py +++ b/TAP/datadictionary.py @@ -186,7 +186,8 @@ def __init__(self, conn, table, connectInfo, **kwargs): # For SQLite the schema is ATTACHed under a name held in # `tap_schema_file`; `tap_schema` is the file path. For # Oracle/Postgres/MySQL `tap_schema` is the schema name itself. - schema_name = self.connectInfo.get("tap_schema_file") \ + schema_name = (self.connectInfo.get("tap_schema_file") + if self.connectInfo.get("dbms") == "sqlite3" else None) \ or self.connectInfo["tap_schema"] sql = "select * from " + schema_name + "." + self.connectInfo["columns_table"] + " where lower(table_name) = " \ diff --git a/TAP/tablevalidator.py b/TAP/tablevalidator.py index 74236b7..5c0bf9c 100644 --- a/TAP/tablevalidator.py +++ b/TAP/tablevalidator.py @@ -38,7 +38,7 @@ def __init__(self, conn, connectInfo=None, debug=0): # For SQLite, `tap_schema` is the FILE PATH and using it directly # in a `SELECT ... FROM .tables` reference produces a SQL # syntax error. Prefer the more-specific key when it's set. - if 'tap_schema_file' in connectInfo: + if connectInfo.get('dbms') == 'sqlite3' and 'tap_schema_file' in connectInfo: self.tap_schema = connectInfo['tap_schema_file'] elif 'tap_schema' in connectInfo: self.tap_schema = connectInfo['tap_schema'] diff --git a/tests/test_tap_schema_name.py b/tests/test_tap_schema_name.py new file mode 100644 index 0000000..d07d55c --- /dev/null +++ b/tests/test_tap_schema_name.py @@ -0,0 +1,68 @@ +"""TAP_SCHEMA's name in SQL: the schema for Oracle, the ATTACH name for SQLite. + +configparam always passes tap_schema_file='TAP_SCHEMA.db' (SQLite's attach +name). Using it for Oracle produced 'FROM TAP_SCHEMA.db.tables' and +ORA-03048 on every query (found with nea-box, 2026-10-05). +""" +import contextlib + +from TAP.datadictionary import dataDictionary +from TAP.tablevalidator import TableValidator + + +class _Cursor: + def __init__(self, log): + self.log = log + self.description = [('column_name',), ('datatype',), ('description',), ('unit',)] + self.arraysize = 100 + + def execute(self, sql, *params): + self.log.append(sql) + + def fetchall(self): + return [('ps',)] + + def fetchmany(self, *a): + return [] + + def close(self): + pass + + +class _Conn: + def __init__(self): + self.log = [] + + def cursor(self): + return _Cursor(self.log) + + +def info(dbms, file_name): + return {'dbms': dbms, 'tap_schema': 'TAP_SCHEMA', 'tap_schema_file': file_name, + 'tables_table': 'tables', 'columns_table': 'columns'} + + +def test_validator_uses_schema_name_for_oracle(): + conn = _Conn() + TableValidator(conn, info('oracle', 'TAP_SCHEMA.db')) + assert conn.log == ['SELECT table_name FROM TAP_SCHEMA.tables'] + + +def test_validator_uses_attach_name_for_sqlite(): + conn = _Conn() + TableValidator(conn, info('sqlite3', 'TS')) + assert conn.log == ['SELECT table_name FROM TS.tables'] + + +def test_data_dictionary_uses_schema_name_for_oracle(): + conn = _Conn() + with contextlib.suppress(Exception): # the fake has no rows; only the SQL matters + dataDictionary(conn, 'ps', info('oracle', 'TAP_SCHEMA.db')) + assert conn.log and conn.log[0].startswith('select * from TAP_SCHEMA.columns ') + + +def test_data_dictionary_uses_attach_name_for_sqlite(): + conn = _Conn() + with contextlib.suppress(Exception): + dataDictionary(conn, 'ps', info('sqlite3', 'TS')) + assert conn.log and conn.log[0].startswith('select * from TS.columns ') From f4ebf354bba61fbf8a5c338be7c97408466a19d1 Mon Sep 17 00:00:00 2001 From: BJ Fulton Date: Tue, 6 Oct 2026 10:39:03 -0600 Subject: [PATCH 15/31] Add compat mode's config side: COMPAT names and the 1.x reader A 1.x TAP.conf ([webserver]/[]/ADQL_*) is translated to the 3.x layout in memory and turns on every NEA compat behavior; a [WEB] config may list them in COMPAT. Unknown names fail at startup. Co-Authored-By: Claude Sonnet 5.5 --- TAP/compat.py | 88 ++++++++++++++++++++++++++ TAP/configparam.py | 4 ++ TAP/tap.py | 3 + tests/test_compat_config.py | 123 ++++++++++++++++++++++++++++++++++++ 4 files changed, 218 insertions(+) create mode 100644 TAP/compat.py create mode 100644 tests/test_compat_config.py diff --git a/TAP/compat.py b/TAP/compat.py new file mode 100644 index 0000000..fadc969 --- /dev/null +++ b/TAP/compat.py @@ -0,0 +1,88 @@ +"""NEA compatibility mode: NEA's pre-3.x responses on the shared code line. + +Compat mode is on when TAP.conf uses the 1.x layout ([webserver], no [WEB]), +which implies every behavior below, or when [WEB] lists COMPAT names. Each +function returns what the default code does unless its behavior is active, +so call sites hold no conditionals, and retiring compat means deleting this +module and compat_vositables.py. See MIGRATING.md for the plan. +""" +import configobj + +NAMES = ('nea-vosi-headers', 'nea-errors', 'nea-tables', 'nea-votable', 'nea-uws') +ALL = frozenset(NAMES) +NONE = frozenset() + +# 1.x [webserver] keys that 3.x reads from [WEB]. +_WEB_KEYS = ('TAP_WORKDIR', 'TAP_WORKURL', 'HTTP_URL', 'HTTP_PORT', 'CGI_PGM', + 'ArraySize', 'INFOMSG') +# 1.x [webserver] keys that 3.x reads from the connection section. +_DB_KEYS_FROM_WEB = ('COOKIENAME', 'RACOL', 'DECCOL') +# 1.x ADQL_* keys and their 3.x [SPTIND] names. +_SPTIND_KEYS = {'ADQL_MODE': 'MODE', 'ADQL_LEVEL': 'LEVEL', 'ADQL_XCOL': 'XCOL', + 'ADQL_YCOL': 'YCOL', 'ADQL_ZCOL': 'ZCOL', + 'ADQL_COLNAME': 'COLNAME', 'ADQL_ENCODING': 'ENCODING'} + + +class CompatConfigError(Exception): + pass + + +def parse_names(value): + """A COMPAT value (configobj gives str or list) -> frozenset of names.""" + if value is None: + return NONE + items = [value] if isinstance(value, str) else list(value) + names = frozenset(v.strip() for v in items if v.strip()) + unknown = sorted(names - ALL) + if unknown: + raise CompatConfigError('unknown COMPAT name(s) %s; valid names: %s' + % (', '.join(unknown), ', '.join(NAMES))) + return names + + +def _find(conf, key): + """A 1.x key wherever configobj filed it: [webserver] first, then any section. + + Keys written after a section header belong to that section, so ADQL_* + lines placed after [oracle] land in [oracle]. + """ + if key in conf['webserver']: + return conf['webserver'][key] + for name in conf.sections: + if key in conf[name]: + return conf[name][key] + return None + + +def translate_legacy(conf): + """A 1.x TAP.conf ([webserver], [], ADQL_*) as an equivalent 3.x one.""" + web = conf['webserver'] + dbms = web.get('DBMS', '') + if dbms not in conf: + raise CompatConfigError('1.x TAP.conf names DBMS=%s but has no [%s] section' + % (dbms, dbms)) + new = configobj.ConfigObj() + new['WEB'] = {k: web[k] for k in _WEB_KEYS if k in web} + db = {'DBMS': dbms} + db.update({k: v for k, v in conf[dbms].items() if not k.startswith('ADQL_')}) + for k in _DB_KEYS_FROM_WEB: + if k in web: + db[k] = web[k] + if dbms == 'sqlite3': + db.setdefault('TAP_SCHEMA_FILE', 'TAP_SCHEMA') # 3.x's SQLite ATTACH name + new['DBMS'] = db + sptind = {} + for old, current in _SPTIND_KEYS.items(): + value = _find(conf, old) + if value is not None: + sptind[current] = value + new['SPTIND'] = sptind + return new + + +def load_config(conf): + """(ConfigObj as read) -> (ConfigObj in 3.x layout, active compat names).""" + if 'WEB' not in conf and 'webserver' in conf: + return translate_legacy(conf), ALL + web = conf['WEB'] if 'WEB' in conf else {} + return conf, parse_names(web.get('COMPAT')) diff --git a/TAP/configparam.py b/TAP/configparam.py index 58701e8..891dfa9 100644 --- a/TAP/configparam.py +++ b/TAP/configparam.py @@ -9,6 +9,8 @@ import configobj +from TAP import compat + class configParam: @@ -71,6 +73,7 @@ def __init__(self, path, **kwargs): raise Exception(self.msg) confobj = configobj.ConfigObj(self.configpath) + confobj, self.compat = compat.load_config(confobj) if self.debug: logging.debug('') @@ -647,6 +650,7 @@ def __init__(self, path, **kwargs): self.connectInfo['columns_table'] = self.columns_table self.connectInfo['keys_table'] = self.keys_table self.connectInfo['key_columns_table'] = self.key_columns_table + self.connectInfo['compat'] = self.compat self.connectInfo['port'] = self.dbport self.connectInfo['socket'] = self.socket self.connectInfo['dbschema'] = self.dbschema diff --git a/TAP/tap.py b/TAP/tap.py index 61213c1..ffc6da8 100755 --- a/TAP/tap.py +++ b/TAP/tap.py @@ -19,6 +19,7 @@ from bs4 import BeautifulSoup from spatial_index import SpatialIndex +from TAP import compat # noqa: F401 (used from Task 3) from TAP.configparam import configParam from TAP.propfilter import propFilter from TAP.tablenames import TableNames @@ -143,6 +144,7 @@ class Tap: uwsheader = '' + compat = frozenset() # active compat names; set once TAP.conf loads def __init__(self, **kwargs): @@ -470,6 +472,7 @@ def __run__(self, **kwargs): self.config = None try: self.config = configParam(self.configpath, instance=self.instance, debug=self.debug) + self.compat = self.config.compat if self.debug: logging.debug('') diff --git a/tests/test_compat_config.py b/tests/test_compat_config.py new file mode 100644 index 0000000..cac9faf --- /dev/null +++ b/tests/test_compat_config.py @@ -0,0 +1,123 @@ +"""Compat mode's config side: COMPAT names and the 1.x TAP.conf translator.""" +import pytest + +from TAP import compat +from TAP.configparam import configParam + +LEGACY_ORACLE = """[webserver] +DBMS=oracle +TAP_WORKDIR={work}/ +TAP_WORKURL=/workspace/ +HTTP_PORT=80 +COOKIENAME=vmexoweb28080 +CGI_PGM=/TAP +RACOL=ra +DECCOL=dec +# Spatial indexing settings +ADQL_MODE=HTM +ADQL_LEVEL=20 +ADQL_XCOL=x +ADQL_YCOL=y +ADQL_ZCOL=z +ADQL_COLNAME=htm20 +ADQL_ENCODING=BASE10 +HTTP_URL=https://exoplanetarchive.ipac.caltech.edu +[oracle] +ServerName=exoops1 +UserID=exo_tap +Password=secret +""" + +CURRENT_ORACLE = """[WEB] +TAP_WORKDIR={work}/ +TAP_WORKURL=/workspace/ +HTTP_URL=https://exoplanetarchive.ipac.caltech.edu +HTTP_PORT=80 +CGI_PGM=/TAP +{compat} +[DBMS] +DBMS=oracle +ServerName=exoops1 +UserID=exo_tap +Password=secret +COOKIENAME=vmexoweb28080 +RACOL=ra +DECCOL=dec +[SPTIND] +MODE=HTM +LEVEL=20 +XCOL=x +YCOL=y +ZCOL=z +COLNAME=htm20 +ENCODING=BASE10 +""" + + +def write(tmp_path, name, text, **fmt): + p = tmp_path / name + p.write_text(text.format(work=tmp_path, **fmt)) + return str(p) + + +def state(cp): + """Everything configParam derived, minus where it came from.""" + out = {k: v for k, v in vars(cp).items() if k not in ('configpath', 'compat', 'connectInfo')} + out['connectInfo'] = {k: v for k, v in cp.connectInfo.items() if k != 'compat'} + return out + + +def test_parse_names_accepts_str_and_list(): + assert compat.parse_names(None) == compat.NONE + assert compat.parse_names('nea-errors') == {'nea-errors'} + assert compat.parse_names(['nea-errors', ' nea-uws ']) == {'nea-errors', 'nea-uws'} + assert compat.parse_names(['', '']) == compat.NONE + + +def test_unknown_compat_name_is_an_error(tmp_path): + with pytest.raises(compat.CompatConfigError, match='nea-eror'): + compat.parse_names(['nea-eror']) + path = write(tmp_path, 'TAP.conf', CURRENT_ORACLE, compat='COMPAT = nea-eror') + with pytest.raises(compat.CompatConfigError): + configParam(path) + + +def test_legacy_oracle_reads_like_its_3x_equivalent(tmp_path): + legacy = configParam(write(tmp_path, 'legacy.conf', LEGACY_ORACLE)) + current = configParam(write(tmp_path, 'current.conf', CURRENT_ORACLE, compat='')) + assert state(legacy) == state(current) + assert legacy.compat == compat.ALL + assert legacy.connectInfo['compat'] == compat.ALL + assert current.compat == compat.NONE + assert current.connectInfo['compat'] == compat.NONE + + +def test_explicit_compat_list(tmp_path): + cp = configParam(write(tmp_path, 'TAP.conf', CURRENT_ORACLE, + compat='COMPAT = nea-errors, nea-uws')) + assert cp.compat == {'nea-errors', 'nea-uws'} + + +def test_adql_keys_after_db_section_are_found(tmp_path): + # configobj files keys that follow [oracle] under [oracle]. + text = LEGACY_ORACLE.replace('ADQL_MODE=HTM\n', '').replace( + 'Password=secret\n', 'Password=secret\nADQL_MODE=HTM\n') + conf = compat.translate_legacy(__import__('configobj').ConfigObj( + write(tmp_path, 'TAP.conf', text))) + assert conf['SPTIND']['MODE'] == 'HTM' + assert 'ADQL_MODE' not in conf['DBMS'] + + +def test_legacy_sqlite_gets_an_attach_name(tmp_path): + text = '[webserver]\nDBMS=sqlite3\nTAP_WORKDIR={work}/\n[sqlite3]\nDB=/x.db\nTAP_SCHEMA=/ts.db\n' + conf = compat.translate_legacy(__import__('configobj').ConfigObj( + write(tmp_path, 'TAP.conf', text))) + assert dict(conf['DBMS']) == {'DBMS': 'sqlite3', 'DB': '/x.db', 'TAP_SCHEMA': '/ts.db', + 'TAP_SCHEMA_FILE': 'TAP_SCHEMA'} + + +def test_legacy_without_its_db_section_is_an_error(tmp_path): + text = '[webserver]\nDBMS=oracle\nTAP_WORKDIR={work}/\n' + with pytest.raises(compat.CompatConfigError, match=r'no \[oracle\] section'): + compat.translate_legacy(__import__('configobj').ConfigObj( + write(tmp_path, 'TAP.conf', text))) From e95ffc600fa461d65a8e7eb64c0e477e296dab27 Mon Sep 17 00:00:00 2001 From: BJ Fulton Date: Tue, 6 Oct 2026 10:42:00 -0600 Subject: [PATCH 16/31] compat: nea-vosi-headers, and a 1.x-config test server Co-Authored-By: Claude Sonnet 5.5 --- TAP/compat.py | 15 +++++++++ TAP/tap.py | 6 +++- tests/_helpers.py | 22 +++++++++++++ tests/conftest.py | 42 ++++++++++++++++++++++++- tests/fixtures/TAP.conf.legacy.template | 23 ++++++++++++++ tests/test_compat_http.py | 22 +++++++++++++ 6 files changed, 128 insertions(+), 2 deletions(-) create mode 100644 tests/fixtures/TAP.conf.legacy.template create mode 100644 tests/test_compat_http.py diff --git a/TAP/compat.py b/TAP/compat.py index fadc969..4155217 100644 --- a/TAP/compat.py +++ b/TAP/compat.py @@ -86,3 +86,18 @@ def load_config(conf): return translate_legacy(conf), ALL web = conf['WEB'] if 'WEB' in conf else {} return conf, parse_names(web.get('COMPAT')) + + +# --- nea-vosi-headers ------------------------------------------------------- + +def vosi_head(active): + """Status line and headers NEA's nph- CGI writes before a VOSI document.""" + if 'nea-vosi-headers' not in active: + return '' + return ('HTTP/1.1 200 OK\r\nContent-type: application/xml\r\n' + 'Connection: close\r\n\r\n') + + +def vosi_tail(active): + """NEA ends its VOSI documents with one more CRLF.""" + return '\r\n' if 'nea-vosi-headers' in active else '' diff --git a/TAP/tap.py b/TAP/tap.py index ffc6da8..ddc8fcc 100755 --- a/TAP/tap.py +++ b/TAP/tap.py @@ -19,7 +19,7 @@ from bs4 import BeautifulSoup from spatial_index import SpatialIndex -from TAP import compat # noqa: F401 (used from Task 3) +from TAP import compat from TAP.configparam import configParam from TAP.propfilter import propFilter from TAP.tablenames import TableNames @@ -3246,6 +3246,7 @@ def __printVosiAvailability__ (self, **kwargs): # { printVosiAvailability # + sys.stdout.write(compat.vosi_head(self.compat)) print ('') print ('') print ('TAP service available.') print ('') + sys.stdout.write(compat.vosi_tail(self.compat)) sys.exit() # @@ -3269,6 +3271,7 @@ def __printVosiCapability__ (self, **kwargs): # { printVosiCapability # + sys.stdout.write(compat.vosi_head(self.compat)) print ('') print ('') print ('') + sys.stdout.write(compat.vosi_tail(self.compat)) sys.exit() # diff --git a/tests/_helpers.py b/tests/_helpers.py index 11873ce..60c32c1 100644 --- a/tests/_helpers.py +++ b/tests/_helpers.py @@ -20,3 +20,25 @@ def row_count_from_ipac(body: str) -> int: """ lines = [L for L in body.splitlines() if L.strip()] return max(0, len(lines) - 4) + + +def raw_get(server: str, path: str) -> bytes: + """GET *path* and return the response bytes exactly as sent (no parsing). + + HTTP libraries reject or repair a response with no status line, which is + the very thing the VOSI tests need to see. + """ + import socket + from urllib.parse import urlsplit + + u = urlsplit(server) + with socket.create_connection((u.hostname, u.port), timeout=30) as s: + s.sendall((f"GET {path} HTTP/1.1\r\nHost: {u.netloc}\r\n" + f"Connection: close\r\n\r\n").encode()) + chunks = [] + while True: + b = s.recv(65536) + if not b: + break + chunks.append(b) + return b"".join(chunks) diff --git a/tests/conftest.py b/tests/conftest.py index e0ab97c..e2e9b20 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -50,7 +50,9 @@ def fixture_root(tmp_path_factory) -> Path: build_test_db.build_all(str(root)) - (root / "workdir").mkdir() + # tap.py resolves a VOSI request's workspace as /TAP and + # fails with a 500 if it is missing; a deployed service always has it. + (root / "workdir" / "TAP").mkdir(parents=True) cgi_dir = root / "cgi-bin" / "TAP" cgi_dir.mkdir(parents=True) shutil.copy(FIXTURES / "nph-tap.py", cgi_dir / "nph-tap.py") @@ -137,6 +139,10 @@ def _run_cgi(self): "CONTENT_LENGTH": str(content_length), "HTTP_HOST": self.headers.get("Host", ""), }) + # A server may carry its own TAP.conf (legacy_tap_server does); + # otherwise the process-wide TAP_CONF set by tap_server applies. + if getattr(self.server, "tap_conf", None): + env["TAP_CONF"] = self.server.tap_conf import subprocess proc = subprocess.Popen( @@ -213,3 +219,37 @@ def tap_server(fixture_root: Path, tap_conf: Path): os.chdir(original_cwd) +@pytest.fixture(scope="session") +def legacy_tap_server(fixture_root: Path, tap_server: str): + """The same SQLite data behind NEA's 1.x TAP.conf layout: compat mode on. + + Depends on tap_server for PYTHONPATH and the working directory it sets. + """ + port = _free_port() + workdir = fixture_root / "workdir-legacy" + (workdir / "TAP").mkdir(parents=True, exist_ok=True) # as in fixture_root + conf = (FIXTURES / "TAP.conf.legacy.template").read_text().format( + TEST_WORKDIR=str(workdir), + TEST_HTTP_HOST="127.0.0.1", + TEST_HTTP_PORT=str(port), + TEST_DB_PATH=str(fixture_root / "test_data.db"), + TEST_TAP_SCHEMA=str(fixture_root / "tap_schema.db"), + ) + conf_path = fixture_root / "TAP.legacy.conf" + conf_path.write_text(conf) + httpd = http.server.HTTPServer(("127.0.0.1", port), _NphCGIHandler) + httpd.allow_reuse_address = True + httpd.nph_script_path = str(fixture_root / "cgi-bin" / "TAP" / "nph-tap.py") + httpd.tap_conf = str(conf_path) + thread = threading.Thread(target=httpd.serve_forever, daemon=True) + thread.start() + for _ in range(50): + try: + with socket.create_connection(("127.0.0.1", port), timeout=0.2): + break + except OSError: + time.sleep(0.05) + try: + yield f"http://127.0.0.1:{port}" + finally: + httpd.shutdown() diff --git a/tests/fixtures/TAP.conf.legacy.template b/tests/fixtures/TAP.conf.legacy.template new file mode 100644 index 0000000..8d414c3 --- /dev/null +++ b/tests/fixtures/TAP.conf.legacy.template @@ -0,0 +1,23 @@ +# NEA's 1.x TAP.conf layout ([webserver] + [sqlite3] + ADQL_*), for the +# compat-mode tests. Same SQLite data as TAP.conf.template; placeholders +# are filled by the legacy_tap_server fixture in tests/conftest.py. +[webserver] +DBMS=sqlite3 +TAP_WORKDIR={TEST_WORKDIR} +TAP_WORKURL=/workspace +HTTP_URL=http://{TEST_HTTP_HOST} +HTTP_PORT={TEST_HTTP_PORT} +CGI_PGM=TAP +RACOL=ra +DECCOL=dec +ADQL_MODE=HTM +ADQL_LEVEL=20 +ADQL_XCOL=x +ADQL_YCOL=y +ADQL_ZCOL=z +ADQL_COLNAME=sptind +ADQL_ENCODING=BASE10 + +[sqlite3] +DB={TEST_DB_PATH} +TAP_SCHEMA={TEST_TAP_SCHEMA} diff --git a/tests/test_compat_http.py b/tests/test_compat_http.py new file mode 100644 index 0000000..87a1400 --- /dev/null +++ b/tests/test_compat_http.py @@ -0,0 +1,22 @@ +"""Compat mode end to end on the SQLite fixture: a 1.x TAP.conf gives NEA's +responses (legacy_tap_server); the 3.x config is unchanged (tap_server).""" +from __future__ import annotations + +from _helpers import raw_get + +VOSI_HEAD = (b"HTTP/1.1 200 OK\r\nContent-type: application/xml\r\n" + b"Connection: close\r\n\r\n") +XML_DECL = b'\n' + + +def test_vosi_documents_carry_neas_headers(legacy_tap_server): + for ep in ("availability", "capabilities"): + raw = raw_get(legacy_tap_server, "/cgi-bin/TAP/nph-tap.py/" + ep) + assert raw.startswith(VOSI_HEAD + XML_DECL), raw[:200] + assert raw.endswith(b"\n\r\n"), raw[-80:] + + +def test_vosi_documents_unchanged_in_default_mode(tap_server): + for ep in ("availability", "capabilities"): + raw = raw_get(tap_server, "/cgi-bin/TAP/nph-tap.py/" + ep) + assert raw.startswith(XML_DECL), raw[:200] From 19a25ed82550cd39ea7b85a66e71dec9cf1d12bb Mon Sep 17 00:00:00 2001 From: BJ Fulton Date: Tue, 6 Oct 2026 10:44:28 -0600 Subject: [PATCH 17/31] compat: nea-errors (VOTable error documents, 400 for unlisted tables) Co-Authored-By: Claude Sonnet 5.5 --- TAP/compat.py | 30 ++++++++++++++++++++++++++++++ TAP/tap.py | 12 +++++++++--- tests/test_compat_http.py | 31 ++++++++++++++++++++++++++++++- tests/test_compat_units.py | 19 +++++++++++++++++++ 4 files changed, 88 insertions(+), 4 deletions(-) create mode 100644 tests/test_compat_units.py diff --git a/TAP/compat.py b/TAP/compat.py index 4155217..5f5f813 100644 --- a/TAP/compat.py +++ b/TAP/compat.py @@ -6,6 +6,8 @@ so call sites hold no conditionals, and retiring compat means deleting this module and compat_vositables.py. See MIGRATING.md for the plan. """ +import html + import configobj NAMES = ('nea-vosi-headers', 'nea-errors', 'nea-tables', 'nea-votable', 'nea-uws') @@ -101,3 +103,31 @@ def vosi_head(active): def vosi_tail(active): """NEA ends its VOSI documents with one more CRLF.""" return '\r\n' if 'nea-vosi-headers' in active else '' + + +# --- nea-errors ------------------------------------------------------------- + +def table_denied_status(active): + """Status for a table outside TAP_SCHEMA: NEA answers 400, the shared line 403.""" + return '400' if 'nea-errors' in active else '403' + + +def error_document(active, errmsg, errcode): + """NEA's error response: a VOTable error document whatever the format. + + Returns the whole response, or None when nea-errors is inactive. Markup + is escaped so the document stays well-formed; quotes are not, matching + NEA's bytes for ordinary messages. + """ + if 'nea-errors' not in active: + return None + return ('HTTP/1.1 %s ERROR\r\n' % errcode + + 'Content-type: application/xml\r\n\r\n' + + '\n' + + '\n' + + '\n' + + '\n' + + html.escape(str(errmsg), quote=False) + '\n' + + '\n' + + '\n' + + '\n') diff --git a/TAP/tap.py b/TAP/tap.py index ddc8fcc..2cc3333 100755 --- a/TAP/tap.py +++ b/TAP/tap.py @@ -1503,7 +1503,7 @@ def __run__(self, **kwargs): try: TableValidator.validate_statement(query_adql, debug=self.debug) except Exception as e: - errcode = '403' if isinstance(e, TableValidationError) else '400' + errcode = compat.table_denied_status(self.compat) if isinstance(e, TableValidationError) else '400' if(self.tapcontext == 'async'): self.phase = 'ERROR' self.__writeAsyncError__(str(e), self.statuspath, @@ -1745,7 +1745,7 @@ def __run__(self, **kwargs): # TableValidationError means the query referenced a table # not in TAP_SCHEMA — access denied (403). All other errors # are bad requests (400). - errcode = '403' if isinstance(e, TableValidationError) else '400' + errcode = compat.table_denied_status(self.compat) if isinstance(e, TableValidationError) else '400' if(self.tapcontext == 'async'): @@ -1806,7 +1806,7 @@ def __run__(self, **kwargs): # TableValidationError means the query referenced a table # not in TAP_SCHEMA — access denied (403). All other errors # are bad requests (400). - errcode = '403' if isinstance(e, TableValidationError) else '400' + errcode = compat.table_denied_status(self.compat) if isinstance(e, TableValidationError) else '400' if(self.tapcontext == 'async'): @@ -2556,6 +2556,12 @@ def __printError__(self, fmt, errmsg, **kwargs): if ('errcode' in kwargs): errcode = kwargs['errcode'] + nea = compat.error_document(self.compat, errmsg, errcode) + if nea is not None: # nea-errors replaces the whole response + sys.stdout.write(nea) + sys.stdout.flush() + sys.exit() + httphdr = "HTTP/1.1 " + str(errcode) + " ERROR\r" print(httphdr) diff --git a/tests/test_compat_http.py b/tests/test_compat_http.py index 87a1400..f864ec4 100644 --- a/tests/test_compat_http.py +++ b/tests/test_compat_http.py @@ -2,7 +2,8 @@ responses (legacy_tap_server); the 3.x config is unchanged (tap_server).""" from __future__ import annotations -from _helpers import raw_get +import requests +from _helpers import raw_get, tap_sync_url VOSI_HEAD = (b"HTTP/1.1 200 OK\r\nContent-type: application/xml\r\n" b"Connection: close\r\n\r\n") @@ -20,3 +21,31 @@ def test_vosi_documents_unchanged_in_default_mode(tap_server): for ep in ("availability", "capabilities"): raw = raw_get(tap_server, "/cgi-bin/TAP/nph-tap.py/" + ep) assert raw.startswith(XML_DECL), raw[:200] + + +DENIED = "Table 'not_a_table' is not available for querying." + + +def _sync(server, query, fmt="csv"): + return requests.post(tap_sync_url(server), data={"query": query, "format": fmt}, + timeout=60) + + +def test_unknown_table_is_neas_400_votable(legacy_tap_server): + r = _sync(legacy_tap_server, "select * from not_a_table") + assert r.status_code == 400 + assert r.headers["Content-Type"] == "application/xml" + assert '\n' + DENIED in r.text + + +def test_bad_adql_is_a_votable_error(legacy_tap_server): + r = _sync(legacy_tap_server, "selec x frm y") + assert r.status_code == 400 + assert r.headers["Content-Type"] == "application/xml" + assert r.text.startswith('\n\n') + assert "\na < b & 'c'\n\n" in doc + assert doc.endswith('\n\n\n') + + +def test_error_document_inactive_is_none(): + assert compat.error_document(compat.NONE, 'x', '400') is None + + +def test_table_denied_status(): + assert compat.table_denied_status(compat.ALL) == '400' + assert compat.table_denied_status(compat.NONE) == '403' From 26e3bc82b9b6f30725f8cc107e7bf5799be0c83b Mon Sep 17 00:00:00 2001 From: BJ Fulton Date: Tue, 6 Oct 2026 10:47:44 -0600 Subject: [PATCH 18/31] compat: nea-votable (application/xml, no non-char FIELD descriptions) Co-Authored-By: Claude Sonnet 5.5 --- TAP/compat.py | 20 ++++++++++++++++++++ TAP/tap.py | 4 ++-- TAP/tapquery.py | 1 + TAP/writeresult.py | 6 ++++-- tests/test_compat_http.py | 18 ++++++++++++++++++ tests/test_compat_units.py | 11 +++++++++++ 6 files changed, 56 insertions(+), 4 deletions(-) diff --git a/TAP/compat.py b/TAP/compat.py index 5f5f813..d71b25c 100644 --- a/TAP/compat.py +++ b/TAP/compat.py @@ -131,3 +131,23 @@ def error_document(active, errmsg, errcode): + '\n' + '\n' + '\n') + + +# --- nea-votable ------------------------------------------------------------ + +def votable_content_type(active): + """Content type for VOTable results: NEA sends application/xml.""" + return 'application/xml' if 'nea-votable' in active else 'text/xml' + + +def field_description(active, desc, coltype): + """NEA's VOTables carry no DESCRIPTION for non-char columns. + + The C writer prints the plain form when a non-char column's + description is empty (writerecsmodule.c, non-char branch), so blanking it + here reproduces NEA's FIELDs without touching the C API. Char columns + already match. + """ + if 'nea-votable' in active and coltype != 'char': + return '' + return desc diff --git a/TAP/tap.py b/TAP/tap.py index 2cc3333..f49e521 100755 --- a/TAP/tap.py +++ b/TAP/tap.py @@ -2491,7 +2491,7 @@ def __getStatus__(self, workdir, workspace, key, param, **kwargs): if(format == 'json'): print("Content-type: application/json\r") elif(format == 'votable'): - print("Content-type: text/xml\r") + print("Content-type: %s\r" % compat.votable_content_type(self.compat)) else: print("Content-type: text/plain\r") print("\r") @@ -2690,7 +2690,7 @@ def __printSyncResult__(self, resultpath, format, **kwargs): if(format == 'json'): print("Content-type: application/json\r") elif(format == 'votable'): - print("Content-type: text/xml\r") + print("Content-type: %s\r" % compat.votable_content_type(self.compat)) else: print("Content-type: text/plain\r") print("\r") diff --git a/TAP/tapquery.py b/TAP/tapquery.py index 47bf0db..76129b4 100644 --- a/TAP/tapquery.py +++ b/TAP/tapquery.py @@ -665,6 +665,7 @@ def __init__(self, **kwargs): maxrec=self.maxrec, arraysize=self.arraysize, coldesc=self.coldesc, + compat=self.connectInfo.get('compat', frozenset()), racol=self.racol, deccol=self.deccol, dbms=self.dbms, \ diff --git a/TAP/writeresult.py b/TAP/writeresult.py index a7a24e1..1d499f1 100644 --- a/TAP/writeresult.py +++ b/TAP/writeresult.py @@ -13,7 +13,7 @@ # import logging -from TAP import writerecs +from TAP import compat, writerecs class writeResult: @@ -153,6 +153,8 @@ def __init__(self, cursor, workdir, dd, **kwargs): if('coldesc' in kwargs): self.coldesc = kwargs['coldesc'] + self.compat = kwargs.get('compat', compat.NONE) + self.arraysize = 10000 if('arraysize' in kwargs): self.arraysize = kwargs['arraysize'] @@ -932,7 +934,7 @@ def __init__(self, cursor, workdir, dd, **kwargs): fmtarr.append(fmt) widtharr.append(width) unitsarr.append(units) - descarr.append(desc) + descarr.append(compat.field_description(self.compat, desc, coltype)) i = i + 1 diff --git a/tests/test_compat_http.py b/tests/test_compat_http.py index f864ec4..52064ff 100644 --- a/tests/test_compat_http.py +++ b/tests/test_compat_http.py @@ -49,3 +49,21 @@ def test_unknown_table_unchanged_in_default_mode(tap_server): r = _sync(tap_server, "select * from not_a_table") assert r.status_code == 403 assert r.headers["Content-Type"] == "text/plain" + + +NONCHAR = "select obsjd, l0propint from data_l0" + + +def test_votable_is_neas_form(legacy_tap_server): + r = _sync(legacy_tap_server, NONCHAR, fmt="votable") + assert r.status_code == 200, r.text[:300] + assert r.headers["Content-Type"] == "application/xml" + assert "" not in r.text + assert '' in r.text + + +def test_votable_unchanged_in_default_mode(tap_server): + r = _sync(tap_server, NONCHAR, fmt="votable") + assert r.status_code == 200, r.text[:300] + assert r.headers["Content-Type"] == "text/xml" + assert "Julian date of observation." in r.text diff --git a/tests/test_compat_units.py b/tests/test_compat_units.py index 4dc0dde..71371df 100644 --- a/tests/test_compat_units.py +++ b/tests/test_compat_units.py @@ -17,3 +17,14 @@ def test_error_document_inactive_is_none(): def test_table_denied_status(): assert compat.table_denied_status(compat.ALL) == '400' assert compat.table_denied_status(compat.NONE) == '403' + + +def test_votable_content_type(): + assert compat.votable_content_type(compat.ALL) == 'application/xml' + assert compat.votable_content_type(compat.NONE) == 'text/xml' + + +def test_field_description_blanks_only_non_char_columns(): + assert compat.field_description(compat.ALL, 'Julian date', 'double') == '' + assert compat.field_description(compat.ALL, 'Program ID', 'char') == 'Program ID' + assert compat.field_description(compat.NONE, 'Julian date', 'double') == 'Julian date' From 45cc47188e82afc6f4943b6cab6e392548f8bde0 Mon Sep 17 00:00:00 2001 From: BJ Fulton Date: Tue, 6 Oct 2026 10:51:18 -0600 Subject: [PATCH 19/31] compat: nea-uws (executionDuration, destruction without Z, NEA content types) Co-Authored-By: Claude Sonnet 5.5 --- TAP/compat.py | 27 +++++++++++++++++++ TAP/tap.py | 15 ++++++----- tests/test_compat_http.py | 55 +++++++++++++++++++++++++++++++++++++- tests/test_compat_units.py | 13 +++++++++ 4 files changed, 102 insertions(+), 8 deletions(-) diff --git a/TAP/compat.py b/TAP/compat.py index d71b25c..c896c87 100644 --- a/TAP/compat.py +++ b/TAP/compat.py @@ -151,3 +151,30 @@ def field_description(active, desc, coltype): if 'nea-votable' in active and coltype != 'char': return '' return desc + + +# --- nea-uws ---------------------------------------------------------------- + +def duration_element(active): + """UWS names it executionDuration; the shared line writes it lowercase.""" + return 'executionDuration' if 'nea-uws' in active else 'executionduration' + + +def uws_key(active, key): + """Status-document element for a job sub-resource key from the URL.""" + return duration_element(active) if key == 'executionduration' else key + + +def destruction_suffix(active): + """NEA writes the destruction time without a trailing 'Z'.""" + return '' if 'nea-uws' in active else 'Z' + + +def uws_content_type(active): + """Content type of the job document: NEA sends application/xml.""" + return 'application/xml' if 'nea-uws' in active else 'text/xml' + + +def async_submit_content_type(active): + """Header line in the 303 after an async submit; NEA sends none.""" + return '' if 'nea-uws' in active else 'Content-Type: text/plain\r\n' diff --git a/TAP/tap.py b/TAP/tap.py index f49e521..2cb0968 100755 --- a/TAP/tap.py +++ b/TAP/tap.py @@ -980,7 +980,7 @@ def __run__(self, **kwargs): self.statdict['starttime'] = job['uws:startTime'] self.statdict['endtime'] = job['uws:endTime'] self.statdict['destruction'] = job['uws:destruction'] - self.statdict['duration'] = job['uws:executionduration'] + self.statdict['duration'] = job['uws:' + compat.duration_element(self.compat)] if self.debug: logging.debug ('') @@ -1075,7 +1075,7 @@ def __run__(self, **kwargs): starttime = stime.strftime('%Y-%m-%dT%H:%M:%S.%f')[:-4] destruction = \ - destructtime.strftime('%Y-%m-%dT%H:%M:%S.%f')[:-4] + 'Z' + destructtime.strftime('%Y-%m-%dT%H:%M:%S.%f')[:-4] + compat.destruction_suffix(self.compat) if self.debug: logging.debug('') @@ -2067,7 +2067,7 @@ def __printStatus__(self, key, retval, outtype, **kwargs): if(outtype == 'xml'): - print("Content-type: text/xml\r") + print("Content-type: %s\r" % compat.uws_content_type(self.compat)) print("\r") print('') @@ -2169,7 +2169,7 @@ def __getStatus__(self, workdir, workspace, key, param, **kwargs): logging.debug('Return status.xml to user and exit.') print("HTTP/1.1 200 OK\r") - print("Content-type: text/xml\r") + print("Content-type: %s\r" % compat.uws_content_type(self.compat)) print("\r") print(data) sys.exit() @@ -2267,7 +2267,7 @@ def __getStatus__(self, workdir, workspace, key, param, **kwargs): # { Single value return # retval = 'None' - keystr = 'uws:' + key + keystr = 'uws:' + compat.uws_key(self.compat, key) outstr = '' if((key == 'phase') @@ -2766,7 +2766,7 @@ def __printAsyncResponse__(self, statusurl, **kwargs): sys.stdout.write('HTTP/1.1 303 See Other\r\n') sys.stdout.write('Location: %s\r\n' % statusurl) - sys.stdout.write('Content-Type: text/plain\r\n') + sys.stdout.write(compat.async_submit_content_type(self.compat)) sys.stdout.write('Content-Length: %d\r\n' % len(body.encode('utf-8'))) sys.stdout.write('Connection: close\r\n') @@ -3048,7 +3048,8 @@ def __writeStatusMsg__(self, statuspath, statdict, param, **kwargs): fp.write(f" {statdict['endtime']:s}\n") - fp.write(f" {statdict['duration']:d}\n") + el = compat.duration_element(self.compat) + fp.write(f" {statdict['duration']:d}\n") if (statdict['destruction'] is None): fp.write(' \n') diff --git a/tests/test_compat_http.py b/tests/test_compat_http.py index 52064ff..184ff9e 100644 --- a/tests/test_compat_http.py +++ b/tests/test_compat_http.py @@ -2,8 +2,11 @@ responses (legacy_tap_server); the 3.x config is unchanged (tap_server).""" from __future__ import annotations +import time +import xml.etree.ElementTree as ET + import requests -from _helpers import raw_get, tap_sync_url +from _helpers import raw_get, tap_async_url, tap_sync_url VOSI_HEAD = (b"HTTP/1.1 200 OK\r\nContent-type: application/xml\r\n" b"Connection: close\r\n\r\n") @@ -67,3 +70,53 @@ def test_votable_unchanged_in_default_mode(tap_server): assert r.status_code == 200, r.text[:300] assert r.headers["Content-Type"] == "text/xml" assert "Julian date of observation." in r.text + + +UWS = "{http://www.ivoa.net/xml/UWS/v1.0}" + + +def _completed_job(server): + """Submit, run and wait; returns (the 303 answering RUN, status URL). + + The 303 to the initial submit (job left PENDING) is written by a different + code path and carries no Content-Type in either mode; the one answering + PHASE=RUN is what ``async_submit_content_type`` controls. + """ + sub = requests.post(tap_async_url(server), + data={"query": "select obsjd from data_l0", "format": "csv"}, + allow_redirects=False, timeout=60) + assert sub.status_code == 303, sub.text[:300] + url = sub.headers["Location"] + run = requests.post(url + "/phase", data={"PHASE": "RUN"}, + allow_redirects=False, timeout=60) + assert run.status_code == 303, run.text[:300] + deadline = time.time() + 30 + while time.time() < deadline: + if requests.get(url + "/phase", timeout=30).text.strip() in ("COMPLETED", "ERROR"): + break + time.sleep(0.2) + return run, url + + +def test_uws_is_neas_form(legacy_tap_server): + run, url = _completed_job(legacy_tap_server) + assert "Content-Type" not in run.headers + r = requests.get(url, timeout=30) + assert r.headers["Content-Type"] == "application/xml" + job = ET.fromstring(r.text) + assert job.find(UWS + "phase").text == "COMPLETED" + assert job.find(UWS + "executionDuration") is not None + assert job.find(UWS + "executionduration") is None + assert not job.find(UWS + "destruction").text.endswith("Z") + dur = requests.get(url + "/executionduration", timeout=30) + assert dur.status_code == 200 and dur.text.strip().isdigit(), dur.text[:200] + + +def test_uws_unchanged_in_default_mode(tap_server): + run, url = _completed_job(tap_server) + assert run.headers["Content-Type"] == "text/plain" + r = requests.get(url, timeout=30) + assert r.headers["Content-Type"] == "text/xml" + job = ET.fromstring(r.text) + assert job.find(UWS + "executionduration") is not None + assert job.find(UWS + "destruction").text.endswith("Z") diff --git a/tests/test_compat_units.py b/tests/test_compat_units.py index 71371df..ccf834f 100644 --- a/tests/test_compat_units.py +++ b/tests/test_compat_units.py @@ -28,3 +28,16 @@ def test_field_description_blanks_only_non_char_columns(): assert compat.field_description(compat.ALL, 'Julian date', 'double') == '' assert compat.field_description(compat.ALL, 'Program ID', 'char') == 'Program ID' assert compat.field_description(compat.NONE, 'Julian date', 'double') == 'Julian date' + + +def test_uws_functions(): + assert compat.duration_element(compat.ALL) == 'executionDuration' + assert compat.duration_element(compat.NONE) == 'executionduration' + assert compat.uws_key(compat.ALL, 'executionduration') == 'executionDuration' + assert compat.uws_key(compat.ALL, 'phase') == 'phase' + assert compat.destruction_suffix(compat.ALL) == '' + assert compat.destruction_suffix(compat.NONE) == 'Z' + assert compat.uws_content_type(compat.ALL) == 'application/xml' + assert compat.uws_content_type(compat.NONE) == 'text/xml' + assert compat.async_submit_content_type(compat.ALL) == '' + assert compat.async_submit_content_type(compat.NONE) == 'Content-Type: text/plain\r\n' From 0c0efb59f43d13368940a7881db830d909e9787c Mon Sep 17 00:00:00 2001 From: BJ Fulton Date: Tue, 6 Oct 2026 10:54:55 -0600 Subject: [PATCH 20/31] compat: nea-tables (NEA's /tables writer, kept verbatim from bbc1108) Co-Authored-By: Claude Sonnet 5.5 --- TAP/compat.py | 11 + TAP/compat_vositables.py | 1702 ++++++++++++++++++++++++++++++++++++ TAP/tap.py | 3 +- tests/test_compat_http.py | 11 + tests/test_compat_units.py | 6 + 5 files changed, 1731 insertions(+), 2 deletions(-) create mode 100644 TAP/compat_vositables.py diff --git a/TAP/compat.py b/TAP/compat.py index c896c87..93a1f9f 100644 --- a/TAP/compat.py +++ b/TAP/compat.py @@ -178,3 +178,14 @@ def uws_content_type(active): def async_submit_content_type(active): """Header line in the 303 after an async submit; NEA sends none.""" return '' if 'nea-uws' in active else 'Content-Type: text/plain\r\n' + + +# --- nea-tables ------------------------------------------------------------- + +def vosi_tables_class(active): + """The /tables writer: NEA's (prod's vositables.py, verbatim) or the shared one.""" + if 'nea-tables' in active: + from TAP.compat_vositables import vosiTables + else: + from TAP.vositables import vosiTables + return vosiTables diff --git a/TAP/compat_vositables.py b/TAP/compat_vositables.py new file mode 100644 index 0000000..26f20b9 --- /dev/null +++ b/TAP/compat_vositables.py @@ -0,0 +1,1702 @@ +# ruff: noqa +# NEA compatibility (nea-tables): NEA prod's vositables.py, kept verbatim from +# commit bbc1108 so /tables matches NEA byte for byte. Selected by +# compat.vosi_tables_class(); delete with compat.py (see MIGRATING.md). + +# Copyright (c) 2020, Caltech IPAC. +# This code is released with a BSD 3-clause license. License information is at +# https://github.com/Caltech-IPAC/nexsciTAP/blob/master/LICENSE + + +import sys +import os +import logging + +import datetime + +import argparse + + +class vosiTables: +# +# { class vosiTables +# + + pid = os.getpid() + + debug = 0 + + debugfname = './vosi.debug' + status = '' + msg = '' + returnMsg = "" + + arraysize = 10000 + + # + # DD columns + # + + dd = None + + conn = None + dbtable = '' + + sql = '' + racol = '' + deccol = '' + + nfetch = 1000 + + outpath = '' + + #ntot = 0 + + format = 'votable' + maxrec = -1 + coldesc = 0 + + nschema = 0 + + schema_namearr = [] + schema_descarr = [] + + + def __init__(self, **kwargs): + + # + # { vosiTables.init method + # + + """ + vosiTables class generates the VOSI compatible XML outputs for TAP + query with endpoint TAP/tables + + Required keyword input parameters: + + connectInfo (python dictionary): + a python dictionary containing info needed to make + a "connection" to a DBMS. + + ourpath(char): output table file path + + Usage: + + vosi = vosiTables (\ + dbms=dbms, + dbserver=dbserver, \ + userid=userid, \ + password=password, \ + outpath=outpath, \ + debug=debug) + """ + + self.dbms = '' + self.dbserver = '' + self.userid = '' + self.password = '' + self.outpath = '' + self.debug = 0 + + self.db = '' + self.tap_schema = '' + + if('debug' in kwargs): + self.debug = kwargs['debug'] + + if self.debug: + logging.debug('') + logging.debug('Enter vosiTables.init') + + if('dbms' in kwargs): + self.dbms = kwargs['dbms'] + + if('dbserver' in kwargs): + self.dbserver = kwargs['dbserver'] + + if('userid' in kwargs): + self.userid = kwargs['userid'] + + if('password' in kwargs): + self.password = kwargs['password'] + + if('outpath' in kwargs): + self.outpath = kwargs['outpath'] + + if('db' in kwargs): + self.db = kwargs['db'] + + if('tap_schema' in kwargs): + self.tap_schema = kwargs['tap_schema'] + + # + # Get keyword parameters + # + + if ((self.dbms is None) or (len(self.dbms) == 0)): + self.msg = 'Database name missing.' + self.status = 'error' + #self.__printVosiError__ (self.msg) + + raise Exception (self.msg) + + if self.debug: + logging.debug('') + logging.debug(f'dbms= {self.dbms:s}') + + + if (self.dbms.lower() == 'oracle'): + + if ((self.dbserver is None) or (len(self.dbserver) == 0)): + self.msg = 'Database server name missing.' + self.status = 'error' + #self.__printVosiError__ (self.msg) + raise Exception (self.msg) + + + if self.debug: + logging.debug('') + logging.debug(f'dbserver= {self.dbserver:s}') + + if ((self.userid is None) or (len(self.userid) == 0)): + self.msg = 'Database userid missing.' + self.status = 'error' + #self.__printVosiError__ (self.msg) + raise Exception (self.msg) + + + if self.debug: + logging.debug('') + logging.debug(f'userid= {self.userid:s}') + + if ((self.password is None) or (len(self.password) == 0)): + self.msg = 'Database password missing.' + self.status = 'error' + #self.__printVosiError__ (self.msg) + raise Exception (self.msg) + + + if self.debug: + logging.debug('') + logging.debug(f'password= {self.password:s}') + + elif (self.dbms.lower() == 'sqlite3'): + + if ((self.db is None) or (len(self.db) == 0)): + self.msg = 'Database name missing.' + self.status = 'error' + #self.__printVosiError__ (self.msg) + raise Exception (self.msg) + + + if self.debug: + logging.debug('') + logging.debug(f'db= {self.db:s}') + + if ((self.tap_schema is None) or (len(self.tap_schema) == 0)): + + self.msg = 'Database table name missing.' + self.status = 'error' + #self.__printVosiError__ (self.msg) + raise Exception (self.msg) + + + if self.debug: + logging.debug('') + logging.debug(f'tap_schema= {self.tap_schema:s}') + + + if ((len(self.outpath) == 0) or (self.outpath is None)): + self.msg = 'Required output path is missing.' + #self.__printVosiError__ (self.msg) + raise Exception (self.msg) + + + if self.debug: + logging.debug('') + logging.debug(f'outpath= {self.outpath:s}') + + # + # { Connect to DBMS + # + if(self.dbms.lower() == 'oracle'): + + import cx_Oracle + try: + self.conn = cx_Oracle.connect(self.userid, + self.password, + self.dbserver) + + if self.debug: + logging.debug('') + logging.debug('connected to Oracle, DB ' + self.dbserver) + + except Exception as e: + + self.status = 'error' + self.msg = 'Failed to connect to cx_Oracle' + + #self.__printVosiError__ (self.msg) + raise Exception (self.msg) + + + + elif (self.dbms.lower() == 'sqlite3'): + + import sqlite3 + + self.db = '' + if('db' in self.connectInfo): + self.db = self.connectInfo['db'] + + if(len(self.db) == 0): + self.msg = 'Failed to retrieve required input parameter'\ + ' [db]' + self.status = 'error' + #self.__printVosiError__ (self.msg) + raise Exception (self.msg) + + + + self.tap_schema = '' + if('tap_schema' in self.connectInfo): + self.tap_schema = self.connectInfo['tap_schema'] + + if(len(self.tap_schema) == 0): + self.msg = 'Failed to retrieve required input parameter'\ + ' [tap_schema]' + self.status = 'error' + #self.__printVosiError__ (self.msg) + raise Exception (self.msg) + + + if self.debug: + logging.debug('') + logging.debug(f'db= {self.db:s}') + logging.debug(f'tap_schema= {self.tap_schema:s}') + + try: + self.conn = sqlite3.connect(self.db) + + if self.debug: + logging.debug('') + logging.debug('connected to SQLite3, database ' + self.db) + + cmd = 'ATTACH DATABASE ? AS TAP_SCHEMA' + + dbspec = (self.tap_schema,) + + if self.debug: + logging.debug('') + logging.debug('cmd: ' + cmd + '(' + self.tap_schema + ')') + + cursor = self.conn.cursor() + + cursor.execute(cmd, dbspec) + + if self.debug: + logging.debug('') + logging.debug('TAP_SCHEMA attached') + + except Exception as e: + + self.status = 'error' + self.msg = 'Failed to connect to SQLite3 databases' + #self.__printVosiError__ (self.msg) + raise Exception (self.msg) + + + else: + self.msg = f'Invalid DBMS: {self.dbms:s}' + self.status = 'error' + #self.__printVosiError__ (self.msg) + raise Exception (self.msg) + + + # + # } end Connect to DBMS + # + + # + # start loop for constructing output: + # + # -- vosi header: tableset tag, + # -- query for schemas: schema tag, + # -- for each schema, query tables: table tag, + # -- for each table, query columns: column tag, + # -- end column tag, + # -- for each table, query foreignkey: foreignkey tag, + # -- end foreignkey tag + # -- end table tag, + # -- end schema tag + # -- end tableset tag + + # + # open ouput file in workdir for vosi table output + # + #vositblpath = self.workdir + '/vositable.xml' + vositblpath = self.outpath + + if self.debug: + logging.debug('') + logging.debug(f'vositblpath = {vositblpath:s}') + + + fp = None + try: + fp = open (vositblpath, 'w') + os.chmod(vositblpath, 0o664) + + except Exception as e: + + if self.debug: + logging.debug('') + logging.debug(f'open voditblpath exception: {str(e):s}') + + self.msg = 'Failed to open/create vositbl file.' + #self.__printVosiError__(self.msg) + raise Exception (self.msg) + + + # + # write vositable header + # + fp.write ("\n") + fp.write(" \n") + fp.flush() + + # + # Submit database query for TAP_SCHEMA.schemas + # + + self.sql = "select schema_name, description from TAP_SCHEMA.schemas order by schema_index" + + if self.debug: + logging.debug('') + logging.debug(f'sql = {self.sql:s}') + logging.debug('call execute sql') + + cursor = self.conn.cursor() + + try: + self.__executeSql__(cursor, self.sql) + + except Exception as e: + + if self.debug: + logging.debug('') + logging.debug(f'executeSql exception: {str(e):s}') + + #self.__printVosiError__ (str(e)) + raise Exception (str(e)) + + + if self.debug: + logging.debug('') + logging.debug('returned executeSql') + + + try: + if self.debug: + logging.debug('') + logging.debug('call getSchemaQueryArr__:') + + self.__getSchemaQueryArr__ (cursor, \ + debug=self.debug) + + if self.debug: + logging.debug('') + logging.debug('returned getSchemaDesc') + + except Exception as e: + + if self.debug: + logging.debug('') + logging.debug(f'writeResult exception: {str(e):s}') + + #self.__printVosiError__ (str(e)) + raise Exception (str(e)) + + + self.nschema = len (self.schema_namearr) + + if self.debug: + logging.debug('') + logging.debug(f'nschema= {self.nschema:d}') + logging.debug('') + logging.debug('schema name and desc') + + for i in range (0,self.nschema): + logging.debug(f'{i:d} schema_name={self.schema_namearr[i]:s}') + logging.debug(f'schema_description={self.schema_descarr[i]:s}') + + for ischema in range (0, self.nschema): + # + # { ischema loop + # + # for each schema: + # 1. write header block for each schema, + # + fp.write(" \n") + fp.write(" " + \ + self.schema_namearr[ischema] + "\n") + fp.write(" " + \ + self.schema_descarr[ischema] + "\n") + fp.flush() + + + # + # 2. execute table sql + # + self.sql = "select table_name, description, table_type " + \ + "from TAP_SCHEMA.tables where schema_name='" + \ + self.schema_namearr[ischema] + "' order by table_index" + + if self.debug: + logging.debug('') + logging.debug(f'ischema= {ischema:d}') + logging.debug(f'schema_name= {self.schema_namearr[i]:s}') + logging.debug('') + logging.debug(f'sql = {self.sql:s}') + logging.debug('call execute sql') + + cursor = self.conn.cursor() + + try: + self.__executeSql__(cursor, self.sql) + + except Exception as e: + + if self.debug: + logging.debug('') + logging.debug(f'executeSql exception: {str(e):s}') + + #self.__printVosiError__ (str(e)) + raise Exception (str(e)) + + + if self.debug: + logging.debug('') + logging.debug('returned executeSql') + logging.debug('') + logging.debug('call getTableQueryArr') + + # + # 3. getSqlresult of table sql + # re-initialize arrays + # + + table_namearr = [] + table_descarr = [] + table_typearr = [] + + ntable = 0 + try: + ntable = self.__getTableQueryArr__(cursor, table_namearr, \ + table_descarr, table_typearr, debug=1) + + except Exception as e: + + if self.debug: + logging.debug('') + logging.debug(f'getTableQueryArr exception: {str(e):s}') + + #self.__printVosiError__ (str(e)) + raise Exception (str(e)) + + + if self.debug: + logging.debug('') + logging.debug('returned getTableQueryArr') + logging.debug(f'ntable= {ntable:d}') + + + for itable in range (0, ntable): + logging.debug ('') + logging.debug (f'itable= {itable:d}') + logging.debug ( + f'table_name= {table_namearr[itable]:s}') + logging.debug ( + f'table_desc= {table_descarr[itable]:s}') + logging.debug ( + f'table_type= {table_typearr[itable]:s}') + + # + # for each table: + # + + for itable in range (0, ntable): + # + # { itable loop + # + # 1. write header block for each table, + # + fp.write("
\n") + fp.write(" " + \ + table_namearr[itable] + "\n") + fp.write(" " + \ + table_descarr[itable] + "\n") + + + # + # 2. execute column sql + # + self.sql = "select column_name, description, unit, ucd, " + \ + "datatype, principal, indexed from TAP_SCHEMA.columns " + \ + "where table_name='" + table_namearr[itable] + \ + "' order by column_index" + + if self.debug: + logging.debug('') + logging.debug(f'sql = {self.sql:s}') + logging.debug('call execute sql') + + cursor = self.conn.cursor() + + try: + self.__executeSql__(cursor, self.sql) + + except Exception as e: + + if self.debug: + logging.debug('') + logging.debug(f'executeSql exception: {str(e):s}') + + #self.__printVosiError__ (str(e)) + raise Exception (str(e)) + + + if self.debug: + logging.debug('') + logging.debug('returned executeSql') + + # + # 3. getSqlresult of column sql and write column result + # + try: + self.__writeOneTableResult__ (cursor, fp, **kwargs) + + except Exception as e: + + if self.debug: + logging.debug('') + logging.debug( + f'writeOneTableResult exception: {str(e):s}') + + #self.__printVosiError__ (str(e)) + raise Exception (str(e)) + + + if self.debug: + logging.debug('') + logging.debug('returned writeOneTableResult') + + # + # 4. execute foreignkey sq + # 5. getSqlresult of foreignkey sql + # -- write foreignkey result + + self.sql = "select key_id as foreignKey, " + \ + "target_table as targetTable, " + \ + "from_column as fromColumn, " + \ + "target_column as targetColumn, description " + \ + "from TAP_SCHEMA.keys " + \ + "NATURAL JOIN TAP_SCHEMA.key_columns " + \ + "where from_table='" + table_namearr[itable] + "'" + + if self.debug: + logging.debug('') + logging.debug(f'foreign key sql:'); + logging.debug(f'sql= {self.sql:s}'); + logging.debug('call execute sql') + + cursor = self.conn.cursor() + + try: + self.__executeSql__(cursor, self.sql) + + except Exception as e: + + if self.debug: + logging.debug('') + logging.debug(f'executeSql exception: {str(e):s}') + + #self.__printVosiError__ (str(e)) + raise Exception (str(e)) + + + if self.debug: + logging.debug('') + logging.debug('returned executeSql') + + # + # 5. getSqlresult of foreignkey sql and write foreignkey result + # + try: + self.__writeForeignKey__ (cursor, fp, **kwargs) + + except Exception as e: + + if self.debug: + logging.debug('') + logging.debug( + f'writeForeignKey exception: {str(e):s}') + + #self.__printVosiError__ (str(e)) + raise Exception (str(e)) + + + + if self.debug: + logging.debug('') + logging.debug('returned writeForeignKey') + + + # + # 6. write end block for each table + # + fp.write("
\n") + fp.flush() + + # + # } end itable loop + # + + # + # 2. write end block for each schema + # + fp.write(" \n") + fp.flush() + + # + # } end ischema loop and write end block for xml file + # + fp.write("\n") + fp.flush() + + # + # } end vosiTables.init + # + + + def __getTableQueryArr__ (self, cursor, table_namearr, table_descarr, \ + table_typearr, **kwargs): + # + # { vosiTable.getTableQueryArr + # + # this query contains only three char columns: + # table_name, description, and table_type; + # + + if self.debug: + logging.debug('') + logging.debug('Enter getTableQueryArr') + + size = 0 + + table_colname = [] + table_dbtype = [] + + nfetch = self.arraysize + cursor.arraysize = nfetch + + if self.debug: + logging.debug('') + logging.debug(f'nfetch = {nfetch:d}') + + + i = 0 + for col in cursor.description: + + # + # { For loop for cursor description: + # cursor description is a list with ncols tuples: + # each tuple + # contains each output colname's name, datatype, + # dbdatatype, size, etc... + # + + if self.debug: + logging.debug('') + logging.debug('get cursor description:') + logging.debug(f'i = {i:d} col = ' + str(col)) + + # + # Extract colname, dbdatatype, size, precision, scale from + # col array + # + colname = str(col[0]).lower() + table_colname.append(colname) + + if self.debug: + logging.debug('') + logging.debug(f'colname(lower) = {colname:s}') + + + dbtype = '' + size = None + + dbdatatypestr = str(col[1]) + + if self.debug: + logging.debug('') + logging.debug(f'dbdatatypestr = {dbdatatypestr:s}') + + ind = dbdatatypestr.find("VARCHAR") + if(ind != -1): + dbtype = 'STRING' + + ind = dbdatatypestr.find("STRING") + if(ind != -1): + dbtype = 'STRING' + + ind = dbdatatypestr.find("NUMBER") + if(ind != -1): + dbtype = 'NUMBER' + + if self.debug: + logging.debug(f'dbtype = {dbtype:s}') + + table_dbtype.append(dbtype) + + i = i + 1 + + # + # } end of for loop for analysing cursor description + # + + # + # Start fetching data lines + # + + if self.debug: + logging.debug('') + logging.debug('Start fetching data:') + + if self.debug: + logging.debug('') + logging.debug(f'nfetch= {nfetch:d}') + + irow = 0 + ntot = 0 + + while True: + + # + # { start of while loop for fetching data lines; + # max 10000 lines at a time + # + + rows = cursor.fetchmany() + + nrec = len(rows) + + if self.debug: + logging.debug(f'nrec = {nrec:d}') + logging.debug('') + + + for ll in range(0, nrec): + + # + # { Beginning ll loop: one row + # + + row = rows[ll] + + if self.debug: + logging.debug ('') + logging.debug (f'len(row)= {len(row):d}') + logging.debug ('') + logging.debug ('row:') + logging.debug(row) + + for i in range(0, len(row)): + + # + # { Beginning i loop: one col + # + data = '' + + if (table_dbtype[i] != 'STRING'): + + data = str(row[i]) + else: + data = row[i] + + if self.debug: + logging.debug (f'i= {i:d}, data= {data:s}') + logging.debug (f'table_colname= {table_colname[i]:s}') + + if (table_colname[i].lower() == 'table_name'): + table_namearr.append (data) + + elif (table_colname[i].lower() == 'description'): + table_descarr.append (data) + + elif (table_colname[i].lower() == 'table_type'): + table_typearr.append (data) + + # + # } end of i loop + # + + irow = irow + 1 + + # + # } end of ll loop + # + if self.debug: + logging.debug('') + logging.debug(table_namearr) + logging.debug(table_descarr) + logging.debug(table_typearr) + logging.debug('----------------------------------------') + + + ntot = ntot + nrec + + if(len(rows) < cursor.arraysize): + break + + if self.debug: + logging.debug ('') + logging.debug (f'ntot= {ntot:d}') + # + # } end while loop for fetching data lines + # + + return (ntot) + + # + # } end vosiTables.getTableQueryArr + # + + + + def __getSchemaQueryArr__ (self, cursor, **kwargs): + # + # { vosiTables.getSchemaQueryArr + # + # this query contains only two columns: + # schema_name and schema_description + # + if self.debug: + logging.debug('') + logging.debug(f'Enter __getSchemaQueryArr__\n') + + size = 0 + nfetch = self.arraysize + cursor.arraysize = nfetch + + if self.debug: + logging.debug('') + logging.debug(f'nfetch= {nfetch:d}') + + + i = 0 + for col in cursor.description: + + # + # { For loop for cursor description: + # cursor description is a list with ncols tuples: + # each tuple # contains each output colname's name, datatype, + # dbdatatype, size, etc... + # + + if self.debug: + logging.debug('') + logging.debug('----------------------------------------------') + logging.debug(f'i = {i:d} col = ' + str(col)) + + # + # Extract colname, dbdatatype, size, precision, scale from + # col array + # + + colname = str(col[0]).lower() + + if self.debug: + logging.debug('') + logging.debug(f'colname(lower) = {colname:s}') + + # + # { Extract dbdatatype, display_size, precision, scale from + # col_array: + # col[1], col[2], col[4], and col[5] + # + + dbdatatype = '' + size = None + precision = None + scale = None + + dbdatatypestr = str(col[1]) + + if self.debug: + logging.debug('') + logging.debug(f'dbdatatypestr = {dbdatatypestr:s}') + + ind = dbdatatypestr.find("VARCHAR") + if(ind != -1): + dbdatatype = 'VARCHAR' + + ind = dbdatatypestr.find("STRING") + if(ind != -1): + dbdatatype = 'STRING' + + size = None + if(col[2] is not None): + + size = col[2] + if(len(colname) > size): + size = len(colname) + + if self.debug: + logging.debug(f'size = {size:d}') + + # + # } end extract dbdatatype, display_size, precision, + # scale from + # col_array: col[1], col[2], col[4], and col[5] + + + dbtype = '' + datatype = '' + fmt = '' + width = 0 + + if(dbdatatype == 'STRING') or (dbdatatype == 'VARCHAR'): + + # + # { dbdatatype == string + # + + width = 0 + if(size is not None): + width = size + + if(len(colname) > width): + width = len(colname) + + datatype = 'char' + dbtype = dbdatatype + fmt = str(width) + 's' + + # + # } end dbdatatype == STRING + # + + else: + # + # { unknown type: default to char + # + + datatype = 'char' + dbtype = dbdatatype + + width = 200 + if(len(colname) > width): + width = len(colname) + fmt = str(width) + 's' + + # + # } end unknown type + # + + if self.debug: + logging.debug('') + logging.debug(f' width = {width:d}') + logging.debug(f' dbtype = {dbtype:s}') + logging.debug(f' datatype = {datatype:s}') + logging.debug(f' fmt = {fmt:s}') + + i = i + 1 + + # + # } end of for loop for analysing cursor description + # + + if self.debug: + logging.debug('') + logging.debug('Start fetching data') + logging.debug('----------------------------------------') + + # + # Start fetching data lines + # + + nfetch = self.arraysize + + cursor.arraysize = nfetch + + irow = 0 + ntot = 0 + + while True: + + # + # { start of while loop for fetching data lines; + # max 10000 lines at a time + # + + rows = cursor.fetchmany() + + nrec = len(rows) + + if self.debug: + logging.debug(f'nrec = {nrec:d}') + logging.debug('') + + + for ll in range(0, nrec): + + # + # { Beginning ll loop: one row + # + + row = rows[ll] + + if self.debug: + logging.debug(row) + logging.debug(f'row[0]= {row[0]:s}') + logging.debug(f'row[1]= {row[1]:s}') + + + self.schema_namearr.append(row[0]) + self.schema_descarr.append(row[1]) + + if self.debug: + logging.debug(row) + logging.debug(self.schema_namearr) + logging.debug(self.schema_descarr) + + irow = irow + 1 + + # + # } end of ll loop + # + + if self.debug: + logging.debug('----------------------------------------') + logging.debug(f'irow= {irow:d}') + + ntot = ntot + nrec + + if(len(rows) < cursor.arraysize): + break + + # + # } end while loop for fetching data lines + # + + return + + # + # } end vosiTables.getSchemaQueryArr + # + + + def __writeOneTableResult__ (self, cursor, fp, **kwargs): + + # + # {vosiTables.writeOneTableResult + # + if self.debug: + logging.debug('') + logging.debug('Enter __writeOneTableResult') + + cursor.arraysize = self.arraysize + + dbtype = [] + + i = 0 + for col in cursor.description: + + # + # { For loop for cursor description: + # cursor description is a list with ncols tuples: + # each tuple + # contains each output colname's name, datatype, + # dbdatatype, size, etc... + # + + if self.debug: + logging.debug('') + logging.debug(f'i = {i:d} col = ' + str(col)) + + # + # Extract colname, dbdatatype, size, precision, scale from + # col array + # + + colname = str(col[0]).lower() + + if self.debug: + logging.debug('') + logging.debug(f'colname(lower) = {colname:s}') + + # + # { Extract dbdatatype, display_size, precision, scale from + # col_array: + # col[1], col[2], col[4], and col[5] + # + + dbdatatype = '' + size = None + + dbdatatypestr = str(col[1]) + + if self.debug: + logging.debug('') + logging.debug(f'dbdatatypestr = {dbdatatypestr:s}') + + ind = dbdatatypestr.find("VARCHAR") + if(ind != -1): + dbdatatype = 'STRING' + + ind = dbdatatypestr.find("STRING") + if(ind != -1): + dbdatatype = 'STRING' + + if self.debug: + logging.debug(f'dbdatatype = {dbdatatype:s}') + + dbtype.append(dbdatatype) + # + # } end extract dbdatatype from col_array: col[1] + # + + i = i + 1 + + # + # } end of for loop for analysing dd's cursor description + # + + # + # Start fetching data lines + # + + nfetch = self.arraysize + cursor.arraysize = nfetch + + irow = 0 + ntot = 0 + + while True: + + # + # { start of while loop for fetching data lines; + # max 10000 lines at a time + # + + rows = cursor.fetchmany() + nrec = len(rows) + + if self.debug: + logging.debug(f'nrec = {nrec:d}') + logging.debug('') + + for ll in range(0, nrec): + + + # + # { Beginning ll loop: one row + # + + row = rows[ll] + + col_name = '' + col_desc = '' + col_unit = '' + col_ucd = '' + col_datatype = '' + col_principal = '' + col_indexed = '' + + data = '' + + for i in range(0, len(row)): + + if (row[i] is None): + data = 'None' + elif (dbtype[i] == 'STRING'): + data = row[i] + else: + data = str(row[i]) + + if self.debug: + logging.debug('') + logging.debug(f'i= {i:d}, data= {data:s}') + + if (i == 0): + col_name = data + elif (i == 1): + col_desc = data + elif (i == 2): + col_unit = data + elif (i == 3): + col_ucd = data + elif (i == 4): + col_datatype = data + elif (i == 5): + col_principal = data + elif (i == 6): + col_indexed = data + + irow = irow + 1 + + if col_datatype == 'timestamp': + col_datatype = 'char' + + + # + # for row, write column block + # + fp.write(" \n") + fp.write(" " + col_name + "\n") + fp.write(" " + \ + "\n") + + if (col_unit != 'None'): + fp.write(" " + col_unit + "\n") + + if (col_ucd != 'None'): + fp.write (" " + col_ucd + "\n") + + fp.write (" " + \ + "" + \ + col_datatype + "\n") + + if (col_principal != 'None' and col_principal != 0 \ + and col_principal != '0'): + fp.write(" " + \ + "principal\n") + + if (col_indexed != 'None' and col_indexed != 0 \ + and col_indexed != '0'): + fp.write(" " + "indexed\n") + + fp.write(" \n") + + + # + # } end of ll loop + # + + ntot = ntot + nrec + + if(len(rows) < cursor.arraysize): + break + + if self.debug: + logging.debug('') + logging.debug(f'ntot= {ntot:d}') + + # + # } end of while loop for fetching data lines + # + + return + + # + # }end vosiTables.writeOneTableResult + # + + + def __writeForeignKey__ (self, cursor, fp, **kwargs): + + # + # {vosiTables.writeForeignKey + # + if self.debug: + logging.debug('') + logging.debug('Enter __writeForeignKey') + + cursor.arraysize = self.arraysize + + dbtype = [] + + i = 0 + for col in cursor.description: + + # + # { For loop for cursor description: + # cursor description is a list with ncols tuples: + # each tuple + # contains each output colname's name, datatype, + # dbdatatype, size, etc... + # + + if self.debug: + logging.debug('') + logging.debug(f'i = {i:d} col = ' + str(col)) + + # + # Extract colname, dbdatatype, size, precision, scale from + # col array + # + + colname = str(col[0]).lower() + + if self.debug: + logging.debug('') + logging.debug(f'colname(lower) = {colname:s}') + + # + # { Extract dbdatatype, display_size, precision, scale from + # col_array: + # col[1], col[2], col[4], and col[5] + # + + dbdatatype = '' + size = None + + dbdatatypestr = str(col[1]) + + if self.debug: + logging.debug('') + logging.debug(f'dbdatatypestr = {dbdatatypestr:s}') + + ind = dbdatatypestr.find("VARCHAR") + if(ind != -1): + dbdatatype = 'STRING' + + ind = dbdatatypestr.find("STRING") + if(ind != -1): + dbdatatype = 'STRING' + + if self.debug: + logging.debug(f'dbdatatype = {dbdatatype:s}') + + dbtype.append(dbdatatype) + # + # } end extract dbdatatype from col_array: col[1] + # + + i = i + 1 + + # + # } end of for loop for analysing dd's cursor description + # + + # + # Start fetching data lines + # + + nfetch = self.arraysize + cursor.arraysize = nfetch + + irow = 0 + ntot = 0 + + while True: + + # + # { start of while loop for fetching data lines; + # max 10000 lines at a time + # + + rows = cursor.fetchmany() + nrec = len(rows) + + if self.debug: + logging.debug(f'nrec = {nrec:d}') + logging.debug('') + + for ll in range(0, nrec): + + + # + # { Beginning ll loop: one row + # + + row = rows[ll] + + keyid = '' + targettbl = '' + fromcol = '' + targetcol= '' + desc = '' + + data = '' + for i in range(0, len(row)): + + if (row[i] is None): + data = 'None' + elif (dbtype[i] == 'STRING'): + data = row[i] + else: + data = str(row[i]) + + if self.debug: + logging.debug('') + logging.debug(f'i= {i:d}, data= {data:s}') + + if (i == 0): + keyid = data + elif (i == 1): + targettbl = data + elif (i == 2): + fromcol = data + elif (i == 3): + targetcol = data + elif (i == 4): + desc = data + + irow = irow + 1 + + # + # for row, write column block + # + fp.write(" \n") + fp.write(" " + targettbl + \ + "\n") + + fp.write(" \n") + fp.write(" " + fromcol + \ + "\n") + fp.write(" " + targetcol + \ + "\n") + fp.write(" \n") + + fp.write(" " + \ + desc + "\n") + fp.write(" \n") + + # + # } end of ll loop + # + + ntot = ntot + nrec + + if(len(rows) < cursor.arraysize): + break + + if self.debug: + logging.debug('') + logging.debug(f'ntot= {ntot:d}') + + # + # } end of while loop for fetching data lines + # + + return + + # + # }end vosiTables.writeOneTableResult + # + + + + + + + + + + + def __executeSql__(self, cursor, sql, **kwargs): + # + # { vosiTables.executeSql + # + + debug = 0 + + if('debug' in kwargs): + debug = kwargs['debug'] + + if self.debug: + logging.debug('') + logging.debug('Enter executeSql') + logging.debug(f'sql:= {sql:s}') + + try: + cursor.execute(sql) + except Exception as e: + + msg = str(e).replace('"', "'") + + if self.debug: + logging.debug('') + logging.debug(f'create table exception: {str(msg):s}') + + # raise Exception(str(e)) + raise Exception(msg) + + # + # } end vosiTables.executeSql + # + + + def __printVosiError__(self, errmsg): + + # + # { + # + + httphdr = "HTTP/1.1 500 ERROR\r" + + #print("HTTP/1.1 200 OK\r") + + print(httphdr) + + print("Content-type: text/xml\r") + print("\r") + + print('') + print('') + print('') + print('') + + print(errmsg) + + print('') + print('') + print('') + + sys.stdout.flush() + sys.exit() + + # + # } end of printError + # + + +# +# } end class vosiTables: +# + +def main(): + + print ('Enter main') + + sys.tracebacklimit = 0 + + parser = argparse.ArgumentParser(description='vosiTables') + + parser.add_argument('--dbms', required=True, + help='database (e.g. oracle, sqlite, etc..) to use.') + + parser.add_argument('--dbserver', required=False, + help='Oracle database server (dbserver) to use.') + + parser.add_argument('--userid', required=False, + help='Oracle dbserver userid (userid) to use.') + + parser.add_argument('--password', required=False, + help='Oracle dbserver (password) to use.') + + parser.add_argument('--db', required=False, + help='Sqlite database name to use.') + + parser.add_argument('--tap_schema', required=False, + help='Sqlite database table name to use.') + + parser.add_argument('--outpath', required=True, + help='outpath for the result.') + + parser.add_argument('--debug', required=False, + help='debugflag: 1/0.') + + args = parser.parse_args() + + dbms = '' + dbserver = '' + userid = '' + password = '' + outpath = '' + debug = 0 + + db = '' + tap_schema = '' + + if (args.dbms is not None): + dbms = args.dbms + print (f'dbms= {dbms:s}') + + if (args.dbserver is not None): + dbserver = args.dbserver + print (f'dbserver= {dbserver:s}') + + if (args.userid is not None): + userid = args.userid + print (f'userid= {userid:s}') + + + if (args.password is not None): + password = args.password + print (f'password= {password:s}') + + + if (args.outpath is not None): + outpath = args.outpath + print (f'outpath= {outpath:s}') + + + if (args.debug is not None): + debug = args.debug + print (f'debug= {debug:s}') + + if (args.db is not None): + db = args.db + print (f'db= {db:s}') + + if (args.tap_schema is not None): + tap_schema = args.tap_schema + print (f'tap_schema= {tap_schema:s}') + + + if debug: + debugfname = './vosi.debug' + + logging.basicConfig(filename=debugfname, \ + level=logging.DEBUG) + + print (f'debug turned on') + + logging.debug('') + logging.debug('debug turned on') + logging.debug(f'dbms= {dbms:s}') + logging.debug(f'dbserver= {dbserver:s}') + logging.debug(f'userid= {userid:s}') + logging.debug(f'password= {password:s}') + logging.debug(f'outpath= {outpath:s}') + logging.debug(f'debug= {debug:s}') + + + vositbl = None + try: + if debug: + logging.debug('') + logging.debug('call vosiTables') + + vositbl = vosiTables (dbms=dbms, \ + dbserver=dbserver, \ + userid=userid, \ + password=password, \ + outpath=outpath, \ + debug=debug) + + if debug: + logging.debug('') + logging.debug('vosiTables initialized') + + print('vosiTables initialized') + + except Exception as e: + + if debug: + logging.debug('') + logging.debug(f'vosiTable.init failed: {str(e):s}') + + print(f'vosiTable.init failed: {str(e):s}') + + +if __name__ == "__main__": + main() diff --git a/TAP/tap.py b/TAP/tap.py index 2cb0968..ec26d68 100755 --- a/TAP/tap.py +++ b/TAP/tap.py @@ -25,7 +25,6 @@ from TAP.tablenames import TableNames from TAP.tablevalidator import TableValidationError, TableValidator from TAP.tapquery import tapQuery -from TAP.vositables import vosiTables class Tap: @@ -3190,7 +3189,7 @@ def __printVosiTables__ (self, connectInfo, vosipath, **kwargs): logging.debug('') logging.debug('call vosiTables') - vosiTables (connectInfo=self.config.connectInfo, dbms=dbms, \ + compat.vosi_tables_class(self.compat) (connectInfo=self.config.connectInfo, dbms=dbms, \ dbserver=dbserver, \ userid=userid, \ password=password, \ diff --git a/tests/test_compat_http.py b/tests/test_compat_http.py index 184ff9e..040ddb7 100644 --- a/tests/test_compat_http.py +++ b/tests/test_compat_http.py @@ -120,3 +120,14 @@ def test_uws_unchanged_in_default_mode(tap_server): job = ET.fromstring(r.text) assert job.find(UWS + "executionduration") is not None assert job.find(UWS + "destruction").text.endswith("Z") + + +def test_tables_is_neas_layout(legacy_tap_server): + r = requests.get(legacy_tap_server + "/cgi-bin/TAP/nph-tap.py/tables", timeout=60) + assert r.status_code == 200, r.text[:300] + + +def test_tables_unchanged_in_default_mode(tap_server): + r = requests.get(tap_server + "/cgi-bin/TAP/nph-tap.py/tables", timeout=60) + assert r.status_code == 200, r.text[:300] + assert "principal" not in r.text diff --git a/tests/test_compat_units.py b/tests/test_compat_units.py index ccf834f..4e25e96 100644 --- a/tests/test_compat_units.py +++ b/tests/test_compat_units.py @@ -41,3 +41,9 @@ def test_uws_functions(): assert compat.uws_content_type(compat.NONE) == 'text/xml' assert compat.async_submit_content_type(compat.ALL) == '' assert compat.async_submit_content_type(compat.NONE) == 'Content-Type: text/plain\r\n' + + +def test_vosi_tables_class(): + from TAP import compat_vositables, vositables + assert compat.vosi_tables_class(compat.ALL) is compat_vositables.vosiTables + assert compat.vosi_tables_class(compat.NONE) is vositables.vosiTables From fc516b2e563eefdfcd86903f553a24e1f0ddceff Mon Sep 17 00:00:00 2001 From: BJ Fulton Date: Tue, 6 Oct 2026 10:58:34 -0600 Subject: [PATCH 21/31] compat: daily deprecation warning, MIGRATING.md, version 3.1.0 Co-Authored-By: Claude Haiku 4.5 --- MIGRATING.md | 51 ++++++++++++++++++++++++++++++++++++++ TAP/compat.py | 28 +++++++++++++++++++++ TAP/tap.py | 1 + setup.py | 2 +- tests/test_compat_units.py | 30 ++++++++++++++++++++++ 5 files changed, 111 insertions(+), 1 deletion(-) create mode 100644 MIGRATING.md diff --git a/MIGRATING.md b/MIGRATING.md new file mode 100644 index 0000000..59fd4f9 --- /dev/null +++ b/MIGRATING.md @@ -0,0 +1,51 @@ +# Migrating from the 1.x TAP.conf layout and NEA compatibility mode + +nexsciTAP 3.1 runs deployments that still use the 1.x `TAP.conf` layout +(`[webserver]`, a section named by `DBMS=`, top-level `ADQL_*` keys). A 1.x +config is translated to the 3.x layout when it's read, and it turns on +**compatibility mode**: the responses that deployment sent under nexsciTAP +1.2. Compatibility mode is temporary. While it's on, the service writes one +line a day to stderr (the web server's error log): + + nexsciTAP: compatibility mode active (...); see MIGRATING.md + +## Compatibility behaviors + +| Name | With it (1.2 behavior) | Without it (3.x behavior) | +|---|---|---| +| `nea-vosi-headers` | `/availability` and `/capabilities` send a status line and headers | the document only | +| `nea-errors` | errors are VOTable documents; a table not in TAP_SCHEMA returns 400 | plain text; 403 | +| `nea-tables` | `/tables` uses VODataService `` elements | per-column ``, ``, ``, ``, `` | +| `nea-votable` | VOTable results are `application/xml`, non-char FIELDs carry no DESCRIPTION | `text/xml`, with descriptions and units | +| `nea-uws` | job documents use `executionDuration`, `application/xml`, no `Z` on `destruction` | `executionduration`, `text/xml`, `Z` | + +## Moving to the 3.x layout + +1. Translate the config, keeping every behavior: in `[WEB]`, list them all. + + | 1.x | 3.x | + |---|---| + | `[webserver]` `TAP_WORKDIR`, `TAP_WORKURL`, `HTTP_URL`, `HTTP_PORT`, `CGI_PGM` | `[WEB]`, same keys | + | `[webserver]` `DBMS` | `[DBMS]` `DBMS` | + | the `[oracle]` / `[sqlite3]` section's keys | `[DBMS]`, same keys | + | `[webserver]` `COOKIENAME`, `RACOL`, `DECCOL` | `[DBMS]`, same keys | + | `ADQL_MODE`, `ADQL_LEVEL`, `ADQL_XCOL`, `ADQL_YCOL`, `ADQL_ZCOL`, `ADQL_COLNAME`, `ADQL_ENCODING` | `[SPTIND]` `MODE`, `LEVEL`, `XCOL`, `YCOL`, `ZCOL`, `COLNAME`, `ENCODING` | + + ```ini + [WEB] + COMPAT = nea-vosi-headers, nea-errors, nea-tables, nea-votable, nea-uws + ``` + Responses don't change. + +2. Remove names from `COMPAT` one at a time, each as an announced change for + that service's users. +3. When `COMPAT` is empty, remove the key. + +## Release plan + +- **3.1.0**: compatibility mode, the 1.x reader, this guide. +- **3.x**: where the shared behavior is to change (for example + `executionDuration` casing and VOTable errors, which IVOA standards + require), the release notes say so before any default changes. +- **4.0.0**: compatibility mode, the `COMPAT` key and the 1.x reader are + removed; a 1.x `TAP.conf` then fails at startup with a pointer here. diff --git a/TAP/compat.py b/TAP/compat.py index 93a1f9f..c534c71 100644 --- a/TAP/compat.py +++ b/TAP/compat.py @@ -6,7 +6,10 @@ so call sites hold no conditionals, and retiring compat means deleting this module and compat_vositables.py. See MIGRATING.md for the plan. """ +import datetime import html +import os +import sys import configobj @@ -189,3 +192,28 @@ def vosi_tables_class(active): else: from TAP.vositables import vosiTables return vosiTables + + +# --- deprecation ------------------------------------------------------------ + +def warn_once_per_day(active, workdir, today=None, stream=None): + """While compat is active, one stderr line (Apache's error_log) per day. + + A stamp file /TAP/.compat-warned-, created exclusively, + marks the day, so concurrent requests warn once. Any failure is + swallowed: the warning must never fail a request. + """ + if not active: + return False + try: + day = (today or datetime.date.today()).isoformat() + stamp = os.path.join(workdir, 'TAP', '.compat-warned-' + day) + os.makedirs(os.path.dirname(stamp), exist_ok=True) + with open(stamp, 'x'): + pass + (stream or sys.stderr).write( + 'nexsciTAP: compatibility mode active (%s); see MIGRATING.md\n' + % ', '.join(sorted(active))) + return True + except Exception: + return False diff --git a/TAP/tap.py b/TAP/tap.py index ec26d68..0713fb1 100755 --- a/TAP/tap.py +++ b/TAP/tap.py @@ -472,6 +472,7 @@ def __run__(self, **kwargs): try: self.config = configParam(self.configpath, instance=self.instance, debug=self.debug) self.compat = self.config.compat + compat.warn_once_per_day(self.compat, self.config.workdir) if self.debug: logging.debug('') diff --git a/setup.py b/setup.py index ed0d920..b660788 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ from setuptools import setup setup(name='nexsciTAP', - version='3.0.1', + version='3.1.0', author='John Good', author_email='jcg@caltech.edu', license='LICENSE', diff --git a/tests/test_compat_units.py b/tests/test_compat_units.py index 4e25e96..d4b1944 100644 --- a/tests/test_compat_units.py +++ b/tests/test_compat_units.py @@ -1,4 +1,7 @@ """compat.py's pure functions, active and inactive.""" +import datetime +import io + from TAP import compat @@ -47,3 +50,30 @@ def test_vosi_tables_class(): from TAP import compat_vositables, vositables assert compat.vosi_tables_class(compat.ALL) is compat_vositables.vosiTables assert compat.vosi_tables_class(compat.NONE) is vositables.vosiTables + + +def test_warning_once_per_day(tmp_path): + out = io.StringIO() + day1, day2 = datetime.date(2026, 10, 6), datetime.date(2026, 10, 7) + assert compat.warn_once_per_day(compat.ALL, str(tmp_path), day1, out) is True + assert compat.warn_once_per_day(compat.ALL, str(tmp_path), day1, out) is False + assert compat.warn_once_per_day(compat.ALL, str(tmp_path), day2, out) is True + lines = out.getvalue().splitlines() + assert len(lines) == 2 + assert lines[0].startswith('nexsciTAP: compatibility mode active (nea-errors, ') + assert lines[0].endswith('see MIGRATING.md') + assert (tmp_path / 'TAP' / '.compat-warned-2026-10-06').exists() + + +def test_no_warning_without_compat(tmp_path): + out = io.StringIO() + assert compat.warn_once_per_day(compat.NONE, str(tmp_path), None, out) is False + assert out.getvalue() == '' and not (tmp_path / 'TAP').exists() + + +def test_warning_never_fails_a_request(tmp_path): + blocker = tmp_path / 'file' # a file where the work dir should be + blocker.write_text('x') + out = io.StringIO() + assert compat.warn_once_per_day(compat.ALL, str(blocker), None, out) is False + assert compat.warn_once_per_day(compat.ALL, None, None, out) is False From 503ca96c9f677bd22c3a8397c0696ee9f49a3d4d Mon Sep 17 00:00:00 2001 From: BJ Fulton Date: Tue, 6 Oct 2026 11:19:18 -0600 Subject: [PATCH 22/31] compat: nea-uws job-run 303 framing matches NEA Under nea-uws the 303 that starts a job carries no Content-Type, Content-Length or Connection line, as bbc1108 prints it; the response ends when the CGI exits. Default mode bytes are unchanged. Co-Authored-By: Claude Sonnet 5.5 --- TAP/compat.py | 14 +++++++++++--- TAP/tap.py | 5 +---- tests/test_compat_http.py | 6 +++++- tests/test_compat_units.py | 8 ++++++-- 4 files changed, 23 insertions(+), 10 deletions(-) diff --git a/TAP/compat.py b/TAP/compat.py index c534c71..d65c8ed 100644 --- a/TAP/compat.py +++ b/TAP/compat.py @@ -178,9 +178,17 @@ def uws_content_type(active): return 'application/xml' if 'nea-uws' in active else 'text/xml' -def async_submit_content_type(active): - """Header line in the 303 after an async submit; NEA sends none.""" - return '' if 'nea-uws' in active else 'Content-Type: text/plain\r\n' +def async_submit_headers(active, body): + """Framing headers of the 303 that starts a job (PHASE=RUN). + + NEA sends none: the response ends when the CGI exits, because the + forked worker holds no stdout. Otherwise: Content-Type, the length of + `body` in bytes, and Connection: close. + """ + if 'nea-uws' in active: + return '' + return ('Content-Type: text/plain\r\nContent-Length: %d\r\n' + 'Connection: close\r\n' % len(body.encode('utf-8'))) # --- nea-tables ------------------------------------------------------------- diff --git a/TAP/tap.py b/TAP/tap.py index 0713fb1..f18af13 100755 --- a/TAP/tap.py +++ b/TAP/tap.py @@ -2766,10 +2766,7 @@ def __printAsyncResponse__(self, statusurl, **kwargs): sys.stdout.write('HTTP/1.1 303 See Other\r\n') sys.stdout.write('Location: %s\r\n' % statusurl) - sys.stdout.write(compat.async_submit_content_type(self.compat)) - sys.stdout.write('Content-Length: %d\r\n' - % len(body.encode('utf-8'))) - sys.stdout.write('Connection: close\r\n') + sys.stdout.write(compat.async_submit_headers(self.compat, body)) sys.stdout.write('\r\n') sys.stdout.write(body) sys.stdout.flush() diff --git a/tests/test_compat_http.py b/tests/test_compat_http.py index 040ddb7..0f55d7a 100644 --- a/tests/test_compat_http.py +++ b/tests/test_compat_http.py @@ -80,7 +80,7 @@ def _completed_job(server): The 303 to the initial submit (job left PENDING) is written by a different code path and carries no Content-Type in either mode; the one answering - PHASE=RUN is what ``async_submit_content_type`` controls. + PHASE=RUN is what ``async_submit_headers`` controls. """ sub = requests.post(tap_async_url(server), data={"query": "select obsjd from data_l0", "format": "csv"}, @@ -101,6 +101,8 @@ def _completed_job(server): def test_uws_is_neas_form(legacy_tap_server): run, url = _completed_job(legacy_tap_server) assert "Content-Type" not in run.headers + assert "Content-Length" not in run.headers + assert "Connection" not in run.headers r = requests.get(url, timeout=30) assert r.headers["Content-Type"] == "application/xml" job = ET.fromstring(r.text) @@ -115,6 +117,8 @@ def test_uws_is_neas_form(legacy_tap_server): def test_uws_unchanged_in_default_mode(tap_server): run, url = _completed_job(tap_server) assert run.headers["Content-Type"] == "text/plain" + assert int(run.headers["Content-Length"]) == len(run.content) + assert run.headers["Connection"] == "close" r = requests.get(url, timeout=30) assert r.headers["Content-Type"] == "text/xml" job = ET.fromstring(r.text) diff --git a/tests/test_compat_units.py b/tests/test_compat_units.py index d4b1944..bcd4c0f 100644 --- a/tests/test_compat_units.py +++ b/tests/test_compat_units.py @@ -42,8 +42,12 @@ def test_uws_functions(): assert compat.destruction_suffix(compat.NONE) == 'Z' assert compat.uws_content_type(compat.ALL) == 'application/xml' assert compat.uws_content_type(compat.NONE) == 'text/xml' - assert compat.async_submit_content_type(compat.ALL) == '' - assert compat.async_submit_content_type(compat.NONE) == 'Content-Type: text/plain\r\n' + body = 'Redirect Location: https://x/\u00e9\n' + assert len(body.encode('utf-8')) != len(body) + assert compat.async_submit_headers(compat.ALL, body) == '' + assert compat.async_submit_headers(compat.NONE, body) == ( + 'Content-Type: text/plain\r\nContent-Length: %d\r\nConnection: close\r\n' + % len(body.encode('utf-8'))) def test_vosi_tables_class(): From 5f03ef4846d066063f9c275b2380e8a89f569548 Mon Sep 17 00:00:00 2001 From: BJ Fulton Date: Tue, 6 Oct 2026 11:35:58 -0600 Subject: [PATCH 23/31] compat: accept executionDuration URL key; tolerant status read; COMPAT only in [WEB] - nea-uws accepts /async//executionDuration as bbc1108 did (compat.uws_url_key) - status.xml is read with either duration spelling, so jobs written under one mode survive a switch to the other (compat.duration_value, compat.uws_key) - COMPAT in any section but [WEB] (including 1.x [webserver]) is a config error Co-Authored-By: Claude Sonnet 5.5 --- TAP/compat.py | 34 +++++++++++++++++++++++++++--- TAP/tap.py | 7 ++++--- tests/test_compat_config.py | 12 +++++++++++ tests/test_compat_http.py | 42 +++++++++++++++++++++++++++++++++++++ tests/test_compat_units.py | 18 ++++++++++++++-- 5 files changed, 105 insertions(+), 8 deletions(-) diff --git a/TAP/compat.py b/TAP/compat.py index d65c8ed..c3c87d6 100644 --- a/TAP/compat.py +++ b/TAP/compat.py @@ -87,6 +87,9 @@ def translate_legacy(conf): def load_config(conf): """(ConfigObj as read) -> (ConfigObj in 3.x layout, active compat names).""" + for name in conf.sections: + if name != 'WEB' and 'COMPAT' in conf[name]: + raise CompatConfigError('COMPAT belongs in [WEB], found in [%s]' % name) if 'WEB' not in conf and 'webserver' in conf: return translate_legacy(conf), ALL web = conf['WEB'] if 'WEB' in conf else {} @@ -163,9 +166,34 @@ def duration_element(active): return 'executionDuration' if 'nea-uws' in active else 'executionduration' -def uws_key(active, key): - """Status-document element for a job sub-resource key from the URL.""" - return duration_element(active) if key == 'executionduration' else key +def uws_key(job, key): + """Element of a parsed status.xml that answers a job sub-resource key. + + The duration is spelled as the document spells it (see duration_value). + """ + if key == 'executionduration' and 'uws:executionDuration' in job: + return 'executionDuration' + return key + + +def uws_url_key(active, key): + """The job sub-resource key from the URL, as the code compares it. + + NEA's /async//executionDuration (UWS spelling) is accepted next to the + lowercase form pyvo uses; the shared line knows only the lowercase one. + """ + if 'nea-uws' in active and key == 'executionDuration': + return 'executionduration' + return key + + +def duration_value(job): + """The duration in a parsed status.xml, whichever spelling wrote it. + + A job written under one mode may be read under the other (jobs live up + to four days), so reading tolerates both; writing uses duration_element. + """ + return job.get('uws:executionDuration', job.get('uws:executionduration')) def destruction_suffix(active): diff --git a/TAP/tap.py b/TAP/tap.py index f18af13..8ca2f97 100755 --- a/TAP/tap.py +++ b/TAP/tap.py @@ -873,7 +873,8 @@ def __run__(self, **kwargs): logging.debug('case: getStatus') try: - self.__getStatus__(self.workdir, self.id, self.statuskey, \ + self.__getStatus__(self.workdir, self.id, + compat.uws_url_key(self.compat, self.statuskey), \ self.param) except Exception as e: @@ -980,7 +981,7 @@ def __run__(self, **kwargs): self.statdict['starttime'] = job['uws:startTime'] self.statdict['endtime'] = job['uws:endTime'] self.statdict['destruction'] = job['uws:destruction'] - self.statdict['duration'] = job['uws:' + compat.duration_element(self.compat)] + self.statdict['duration'] = compat.duration_value(job) if self.debug: logging.debug ('') @@ -2267,7 +2268,7 @@ def __getStatus__(self, workdir, workspace, key, param, **kwargs): # { Single value return # retval = 'None' - keystr = 'uws:' + compat.uws_key(self.compat, key) + keystr = 'uws:' + compat.uws_key(job, key) outstr = '' if((key == 'phase') diff --git a/tests/test_compat_config.py b/tests/test_compat_config.py index cac9faf..e8affc1 100644 --- a/tests/test_compat_config.py +++ b/tests/test_compat_config.py @@ -121,3 +121,15 @@ def test_legacy_without_its_db_section_is_an_error(tmp_path): with pytest.raises(compat.CompatConfigError, match=r'no \[oracle\] section'): compat.translate_legacy(__import__('configobj').ConfigObj( write(tmp_path, 'TAP.conf', text))) + + +def test_compat_outside_web_is_an_error(tmp_path): + text = CURRENT_ORACLE.replace('Password=secret\n', 'Password=secret\nCOMPAT = nea-uws\n') + with pytest.raises(compat.CompatConfigError, match=r'\[DBMS\]'): + configParam(write(tmp_path, 'TAP.conf', text, compat='')) + + +def test_compat_in_legacy_webserver_is_an_error(tmp_path): + text = LEGACY_ORACLE.replace('DBMS=oracle\n', 'DBMS=oracle\nCOMPAT=nea-uws\n', 1) + with pytest.raises(compat.CompatConfigError, match=r'\[webserver\]'): + configParam(write(tmp_path, 'TAP.conf', text)) diff --git a/tests/test_compat_http.py b/tests/test_compat_http.py index 0f55d7a..b49556b 100644 --- a/tests/test_compat_http.py +++ b/tests/test_compat_http.py @@ -112,6 +112,9 @@ def test_uws_is_neas_form(legacy_tap_server): assert not job.find(UWS + "destruction").text.endswith("Z") dur = requests.get(url + "/executionduration", timeout=30) assert dur.status_code == 200 and dur.text.strip().isdigit(), dur.text[:200] + camel = requests.get(url + "/executionDuration", timeout=30) + assert camel.status_code == 200 and camel.text.strip().isdigit(), camel.text[:200] + assert camel.text == dur.text def test_uws_unchanged_in_default_mode(tap_server): @@ -126,6 +129,45 @@ def test_uws_unchanged_in_default_mode(tap_server): assert job.find(UWS + "destruction").text.endswith("Z") +def test_camelcase_duration_key_unchanged_in_default_mode(tap_server): + run, url = _completed_job(tap_server) + r = requests.get(url + "/executionDuration", timeout=30) + assert r.status_code == 200 + assert r.headers["Content-Type"] == "text/plain" + assert r.text == "key executionDuration is not a valid key\n" + + +def _respell_duration(fixture_root, url, old, new): + """Rewrite one job's status.xml as if the other mode had written it.""" + jobid = url.rstrip("/").rsplit("/", 1)[1] + paths = list(fixture_root.glob("workdir*/**/%s/status.xml" % jobid)) + assert len(paths) == 1, paths + text = paths[0].read_text() + assert "uws:" + old in text + paths[0].write_text(text.replace("uws:" + old, "uws:" + new)) + + +def _read_back(url): + whole = requests.get(url, timeout=30) + assert whole.status_code == 200 and "COMPLETED<" in whole.text, whole.text[:300] + dur = requests.get(url + "/executionduration", timeout=30) + assert dur.status_code == 200 and dur.text.strip().isdigit(), dur.text[:300] + phase = requests.get(url + "/phase", timeout=30) + assert phase.text.strip() == "COMPLETED", phase.text[:300] + + +def test_status_written_by_default_mode_reads_in_compat(legacy_tap_server, fixture_root): + _, url = _completed_job(legacy_tap_server) + _respell_duration(fixture_root, url, "executionDuration", "executionduration") + _read_back(url) + + +def test_status_written_by_compat_reads_in_default_mode(tap_server, fixture_root): + _, url = _completed_job(tap_server) + _respell_duration(fixture_root, url, "executionduration", "executionDuration") + _read_back(url) + + def test_tables_is_neas_layout(legacy_tap_server): r = requests.get(legacy_tap_server + "/cgi-bin/TAP/nph-tap.py/tables", timeout=60) assert r.status_code == 200, r.text[:300] diff --git a/tests/test_compat_units.py b/tests/test_compat_units.py index bcd4c0f..7059818 100644 --- a/tests/test_compat_units.py +++ b/tests/test_compat_units.py @@ -36,8 +36,14 @@ def test_field_description_blanks_only_non_char_columns(): def test_uws_functions(): assert compat.duration_element(compat.ALL) == 'executionDuration' assert compat.duration_element(compat.NONE) == 'executionduration' - assert compat.uws_key(compat.ALL, 'executionduration') == 'executionDuration' - assert compat.uws_key(compat.ALL, 'phase') == 'phase' + assert compat.uws_key({'uws:executionDuration': '1'}, 'executionduration') == 'executionDuration' + assert compat.uws_key({'uws:executionduration': '1'}, 'executionduration') == 'executionduration' + assert compat.uws_key({'uws:executionDuration': '1'}, 'phase') == 'phase' + assert compat.uws_url_key(compat.ALL, 'executionDuration') == 'executionduration' + assert compat.uws_url_key(compat.ALL, 'executionduration') == 'executionduration' + assert compat.uws_url_key(compat.ALL, 'phase') == 'phase' + assert compat.uws_url_key(compat.NONE, 'executionDuration') == 'executionDuration' + assert compat.uws_url_key(compat.NONE, 'executionduration') == 'executionduration' assert compat.destruction_suffix(compat.ALL) == '' assert compat.destruction_suffix(compat.NONE) == 'Z' assert compat.uws_content_type(compat.ALL) == 'application/xml' @@ -81,3 +87,11 @@ def test_warning_never_fails_a_request(tmp_path): out = io.StringIO() assert compat.warn_once_per_day(compat.ALL, str(blocker), None, out) is False assert compat.warn_once_per_day(compat.ALL, None, None, out) is False + + +def test_duration_value_reads_either_spelling(): + lower = {'uws:executionduration': '30'} + camel = {'uws:executionDuration': '30'} + assert compat.duration_value(lower) == '30' + assert compat.duration_value(camel) == '30' + assert compat.duration_value({}) is None From cd6eb537db2e638f9056aa33aeadabc2b41ef188 Mon Sep 17 00:00:00 2001 From: BJ Fulton Date: Tue, 6 Oct 2026 11:36:43 -0600 Subject: [PATCH 24/31] docs: MIGRATING.md matches compat behavior; tablevalidator comment Co-Authored-By: Claude Sonnet 5.5 --- MIGRATING.md | 24 +++++++++++++++++++++--- TAP/tablevalidator.py | 3 ++- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/MIGRATING.md b/MIGRATING.md index 59fd4f9..417b01f 100644 --- a/MIGRATING.md +++ b/MIGRATING.md @@ -16,8 +16,8 @@ line a day to stderr (the web server's error log): | `nea-vosi-headers` | `/availability` and `/capabilities` send a status line and headers | the document only | | `nea-errors` | errors are VOTable documents; a table not in TAP_SCHEMA returns 400 | plain text; 403 | | `nea-tables` | `/tables` uses VODataService `` elements | per-column ``, ``, ``, ``, `` | -| `nea-votable` | VOTable results are `application/xml`, non-char FIELDs carry no DESCRIPTION | `text/xml`, with descriptions and units | -| `nea-uws` | job documents use `executionDuration`, `application/xml`, no `Z` on `destruction` | `executionduration`, `text/xml`, `Z` | +| `nea-votable` | VOTable results are `application/xml`, non-char FIELDs carry no DESCRIPTION (units are still written) | `text/xml`, with descriptions on every FIELD | +| `nea-uws` | job documents use `executionDuration`, `application/xml`, no `Z` on `destruction`; the 303 that starts a job (`PHASE=RUN`) carries no `Content-Type`, `Content-Length` or `Connection` lines, because the response ends when the CGI exits; `/async//executionDuration` (camelCase) is accepted as well as `/executionduration` | `executionduration`, `text/xml`, `Z`; the 303 carries `Content-Type: text/plain`, `Content-Length` and `Connection: close`; only `/executionduration` is a valid key | ## Moving to the 3.x layout @@ -25,7 +25,7 @@ line a day to stderr (the web server's error log): | 1.x | 3.x | |---|---| - | `[webserver]` `TAP_WORKDIR`, `TAP_WORKURL`, `HTTP_URL`, `HTTP_PORT`, `CGI_PGM` | `[WEB]`, same keys | + | `[webserver]` `TAP_WORKDIR`, `TAP_WORKURL`, `HTTP_URL`, `HTTP_PORT`, `CGI_PGM`, `ArraySize`, `INFOMSG` | `[WEB]`, same keys | | `[webserver]` `DBMS` | `[DBMS]` `DBMS` | | the `[oracle]` / `[sqlite3]` section's keys | `[DBMS]`, same keys | | `[webserver]` `COOKIENAME`, `RACOL`, `DECCOL` | `[DBMS]`, same keys | @@ -41,6 +41,24 @@ line a day to stderr (the web server's error log): that service's users. 3. When `COMPAT` is empty, remove the key. +## Known differences from 1.2 + +Compatibility mode reproduces 1.2's responses, with these exceptions: + +- Errors raised before `TAP.conf` is read (a missing or broken `TAP.conf`, a + malformed request path) use the 3.x error format, since the mode isn't known + yet. +- `/availability` and `/capabilities` create a workspace directory per + request, so workspace cleanup must cover them. +- The async job document escapes quotes in the query text as `'`; it + parses to the same XML. +- A job's `status.xml` written under one mode is read correctly under the + other (either spelling of the duration element), so switching `nea-uws` off + doesn't break jobs in flight. + +`COMPAT` is read only from `[WEB]`; placing it in any other section, including +a 1.x `[webserver]`, is a configuration error. + ## Release plan - **3.1.0**: compatibility mode, the 1.x reader, this guide. diff --git a/TAP/tablevalidator.py b/TAP/tablevalidator.py index 5c0bf9c..fa80940 100644 --- a/TAP/tablevalidator.py +++ b/TAP/tablevalidator.py @@ -37,7 +37,8 @@ def __init__(self, conn, connectInfo=None, debug=0): # there's no ATTACH and `tap_schema` is the schema name itself. # For SQLite, `tap_schema` is the FILE PATH and using it directly # in a `SELECT ... FROM .tables` reference produces a SQL - # syntax error. Prefer the more-specific key when it's set. + # syntax error. So `tap_schema_file` is used only for SQLite + # (dbms == 'sqlite3'); other databases use `tap_schema`. if connectInfo.get('dbms') == 'sqlite3' and 'tap_schema_file' in connectInfo: self.tap_schema = connectInfo['tap_schema_file'] elif 'tap_schema' in connectInfo: From f6a1bd1afa7238bf32304e5bf6c4b6cfde90a7ab Mon Sep 17 00:00:00 2001 From: BJ Fulton Date: Tue, 6 Oct 2026 11:57:34 -0600 Subject: [PATCH 25/31] test: use generic host and cookie names in compat tests Co-Authored-By: Claude Opus 5.5 --- tests/test_compat_config.py | 8 ++++---- tests/test_tap_schema_name.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_compat_config.py b/tests/test_compat_config.py index e8affc1..ef749d4 100644 --- a/tests/test_compat_config.py +++ b/tests/test_compat_config.py @@ -9,7 +9,7 @@ TAP_WORKDIR={work}/ TAP_WORKURL=/workspace/ HTTP_PORT=80 -COOKIENAME=vmexoweb28080 +COOKIENAME=tapcookie CGI_PGM=/TAP RACOL=ra DECCOL=dec @@ -23,7 +23,7 @@ ADQL_ENCODING=BASE10 HTTP_URL=https://exoplanetarchive.ipac.caltech.edu [oracle] -ServerName=exoops1 +ServerName=tapdb UserID=exo_tap Password=secret """ @@ -37,10 +37,10 @@ {compat} [DBMS] DBMS=oracle -ServerName=exoops1 +ServerName=tapdb UserID=exo_tap Password=secret -COOKIENAME=vmexoweb28080 +COOKIENAME=tapcookie RACOL=ra DECCOL=dec [SPTIND] diff --git a/tests/test_tap_schema_name.py b/tests/test_tap_schema_name.py index d07d55c..87e34c0 100644 --- a/tests/test_tap_schema_name.py +++ b/tests/test_tap_schema_name.py @@ -2,7 +2,7 @@ configparam always passes tap_schema_file='TAP_SCHEMA.db' (SQLite's attach name). Using it for Oracle produced 'FROM TAP_SCHEMA.db.tables' and -ORA-03048 on every query (found with nea-box, 2026-10-05). +ORA-03048 on every query (found by comparing against a production deployment, 2026-10-05). """ import contextlib From 8cf8f412de025b3bc6974360337722dbf79cd150 Mon Sep 17 00:00:00 2001 From: BJ Fulton Date: Tue, 6 Oct 2026 13:10:19 -0600 Subject: [PATCH 26/31] Declare runtime dependencies in setup.py sqlparse, astropy, beautifulsoup4, lxml and xmltodict are needed at run time but were listed only in requirements-test.txt, so CI (which installs with --no-deps) passed while a real pip install produced a service whose async jobs all failed: BeautifulSoup asks for the lxml parser by name. Database drivers stay undeclared; each deployment installs its own. tests/test_install_requires.py fails when TAP imports a package that setup.py does not declare. Co-Authored-By: Claude Opus 5.5 --- requirements-test.txt | 8 ++--- setup.py | 6 +++- tests/test_install_requires.py | 59 ++++++++++++++++++++++++++++++++++ 3 files changed, 68 insertions(+), 5 deletions(-) create mode 100644 tests/test_install_requires.py diff --git a/requirements-test.txt b/requirements-test.txt index 22dfd2b..f8c7ea5 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -5,10 +5,10 @@ pytest>=7 ruff==0.15.2 requests -# Runtime deps. These are not declared in setup.py's install_requires (a -# separate cleanup) but the package imports them. The CI install uses -# --no-deps for nexsciTAP itself (to avoid the spatial_index PyPI hole), -# so we list everything actually-needed here. +# Runtime deps, repeated from setup.py's install_requires: the CI install +# uses --no-deps for nexsciTAP itself (to avoid the spatial_index PyPI +# hole), so they are listed here too. tests/test_install_requires.py +# keeps setup.py complete. configobj sqlparse astropy diff --git a/setup.py b/setup.py index b660788..c31d093 100644 --- a/setup.py +++ b/setup.py @@ -13,6 +13,10 @@ description='NExScI VO Table Access Protocol (TAP) web service', long_description=open('README.md').read(), ext_modules=[Extension('TAP/writerecs', ['TAP/writerecsmodule.c'])], - install_requires=['ADQL', 'spatial_index', 'configobj'], + # Database drivers (cx_Oracle, psycopg2, mysql-connector) are not listed: + # each deployment installs the one for its DBMS. lxml is not imported, + # but BeautifulSoup is asked for it by name to read UWS job documents. + install_requires=['ADQL', 'spatial_index', 'configobj', 'sqlparse', + 'astropy', 'beautifulsoup4', 'lxml', 'xmltodict'], packages=['TAP'] ) diff --git a/tests/test_install_requires.py b/tests/test_install_requires.py new file mode 100644 index 0000000..2b22fa2 --- /dev/null +++ b/tests/test_install_requires.py @@ -0,0 +1,59 @@ +"""setup.py declares every package the TAP code needs at run time. + +CI installs nexsciTAP with --no-deps and gets its runtime packages from +requirements-test.txt, so a package missing from install_requires passes +CI and only fails on a real `pip install`. Missing lxml did exactly that: +sync queries worked and every async job returned 500. +""" +from __future__ import annotations + +import ast +import re +import sys +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parent.parent + +# Import name -> distribution name, where they differ. +DIST_NAME = {'bs4': 'beautifulsoup4'} + +# Database drivers are imported only for the DBMS a deployment configures. +OPTIONAL = {'cx_Oracle', 'psycopg2', 'mysql'} + +# Not imported, but named as a parser: BeautifulSoup(data, 'lxml'). +NAMED_ONLY = {'lxml'} + + +def _install_requires() -> set[str]: + tree = ast.parse((REPO_ROOT / 'setup.py').read_text()) + for node in ast.walk(tree): + if isinstance(node, ast.keyword) and node.arg == 'install_requires': + reqs = ast.literal_eval(node.value) + return {re.split(r'[\s;\[<>=!~]', r, maxsplit=1)[0] for r in reqs} + raise AssertionError('setup.py has no install_requires') + + +def _third_party_imports() -> set[str]: + found = set() + for path in (REPO_ROOT / 'TAP').glob('*.py'): + for node in ast.walk(ast.parse(path.read_text())): + if isinstance(node, ast.Import): + names = [a.name for a in node.names] + elif isinstance(node, ast.ImportFrom) and node.level == 0: + names = [node.module or ''] + else: + continue + for name in names: + top = name.split('.')[0] + if top != 'TAP' and top not in sys.stdlib_module_names: + found.add(top) + return found + + +@pytest.mark.skipif(sys.version_info < (3, 10), reason='needs sys.stdlib_module_names') +def test_every_runtime_import_is_declared(): + needed = {DIST_NAME.get(m, m) for m in _third_party_imports() - OPTIONAL} | NAMED_ONLY + missing = needed - _install_requires() + assert not missing, f'add to install_requires in setup.py: {sorted(missing)}' From 426c999a0339ef7ba39f14dac8a0f005da09ab1d Mon Sep 17 00:00:00 2001 From: BJ Fulton Date: Tue, 6 Oct 2026 14:28:13 -0600 Subject: [PATCH 27/31] Keep every 1.x [webserver] key when translating to 3.x translate_legacy copied only COOKIENAME, RACOL and DECCOL from [webserver] to the connection section, so a 1.x config's PROPFILTER, ACCESS_TBL, USERS_TBL, FILEID and ACCESSID were dropped and the proprietary filter was silently off. Every [webserver] key that is not a [WEB] or ADQL_* key now goes to the connection section. Co-Authored-By: Claude Opus 5.5 --- TAP/compat.py | 12 +++++++----- tests/test_compat_config.py | 25 +++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/TAP/compat.py b/TAP/compat.py index f33401f..965f3f6 100644 --- a/TAP/compat.py +++ b/TAP/compat.py @@ -20,8 +20,6 @@ # 1.x [webserver] keys that 3.x reads from [WEB]. _WEB_KEYS = ('TAP_WORKDIR', 'TAP_WORKURL', 'HTTP_URL', 'HTTP_PORT', 'CGI_PGM', 'ArraySize', 'INFOMSG') -# 1.x [webserver] keys that 3.x reads from the connection section. -_DB_KEYS_FROM_WEB = ('COOKIENAME', 'RACOL', 'DECCOL') # 1.x ADQL_* keys and their 3.x [SPTIND] names. _SPTIND_KEYS = {'ADQL_MODE': 'MODE', 'ADQL_LEVEL': 'LEVEL', 'ADQL_XCOL': 'XCOL', 'ADQL_YCOL': 'YCOL', 'ADQL_ZCOL': 'ZCOL', @@ -70,9 +68,13 @@ def translate_legacy(conf): new['WEB'] = {k: web[k] for k in _WEB_KEYS if k in web} db = {'DBMS': dbms} db.update({k: v for k, v in conf[dbms].items() if not k.startswith('ADQL_')}) - for k in _DB_KEYS_FROM_WEB: - if k in web: - db[k] = web[k] + # Every other [webserver] key goes to the connection section, where 3.x + # reads COOKIENAME, RACOL, DECCOL and the proprietary-filter keys + # (PROPFILTER, ACCESS_TBL, USERS_TBL, FILEID, ACCESSID). Nothing is + # dropped: a lost PROPFILTER would serve proprietary rows to anyone. + for k, v in web.items(): + if k not in _WEB_KEYS and k != 'DBMS' and not k.startswith('ADQL_'): + db[k] = v if dbms == 'sqlite3': db.setdefault('TAP_SCHEMA_FILE', 'TAP_SCHEMA') # 3.x's SQLite ATTACH name new['DBMS'] = db diff --git a/tests/test_compat_config.py b/tests/test_compat_config.py index ef749d4..acc7ad5 100644 --- a/tests/test_compat_config.py +++ b/tests/test_compat_config.py @@ -92,6 +92,31 @@ def test_legacy_oracle_reads_like_its_3x_equivalent(tmp_path): assert current.connectInfo['compat'] == compat.NONE +PROPRIETARY_KEYS = ('ACCESS_TBL=access_tbl\nUSERS_TBL=users_tbl\nFILEID=fileid\n' + 'ACCESSID=accessid\nPROPFILTER=koa\n') + + +def test_legacy_proprietary_filter_keys_are_kept(tmp_path): + # A 1.x config keeps the proprietary-filter keys in [webserver]; 3.x reads + # them from the connection section. Dropping them serves proprietary rows. + legacy = configParam(write(tmp_path, 'legacy.conf', LEGACY_ORACLE.replace( + 'CGI_PGM=/TAP\n', 'CGI_PGM=/TAP\n' + PROPRIETARY_KEYS))) + current = configParam(write(tmp_path, 'current.conf', CURRENT_ORACLE.replace( + 'DECCOL=dec\n', 'DECCOL=dec\n' + PROPRIETARY_KEYS), compat='')) + assert legacy.propfilter == 'koa' + assert (legacy.accesstbl, legacy.usertbl, legacy.fileid, legacy.accessid) == \ + ('access_tbl', 'users_tbl', 'fileid', 'accessid') + assert state(legacy) == state(current) + + +def test_every_legacy_webserver_key_is_kept(tmp_path): + # Keys the translator has no list entry for still reach configParam. + text = LEGACY_ORACLE.replace('CGI_PGM=/TAP\n', 'CGI_PGM=/TAP\nSOME_FUTURE_KEY=x\n') + conf = compat.translate_legacy(__import__('configobj').ConfigObj( + write(tmp_path, 'TAP.conf', text))) + assert conf['DBMS']['SOME_FUTURE_KEY'] == 'x' + + def test_explicit_compat_list(tmp_path): cp = configParam(write(tmp_path, 'TAP.conf', CURRENT_ORACLE, compat='COMPAT = nea-errors, nea-uws')) From 3fd38f4bd497858e23f7ea6419132041b246b9d8 Mon Sep 17 00:00:00 2001 From: BJ Fulton Date: Tue, 6 Oct 2026 15:09:42 -0600 Subject: [PATCH 28/31] UWS: delete jobs, 404 unknown jobs, executionDuration, consistent values taplint (E-UWS-DENO, -HTDE, -JDED, -JDDE) reports four async problems: - Deleting a job did nothing: POST ACTION=DELETE returned the job, and HTTP DELETE was a 500 because FieldStorage raises 'not indexable' on a request without form fields. Both now stop a running job, remove its workspace and answer 303 to the job list; the job is then 404. - An unknown job id was a 500; it is 404. Job ids must match the workspace name pattern (tap_...), so no id reaches outside /TAP. - The job document wrote ; UWS names it executionDuration. Reading accepts both, for jobs written earlier. - An unset value read back as the text 'None' (/destruction before a job runs) while the job document had it empty; both are now empty. GET on the job list returned a new job; it now returns an empty , since clients follow the deletion redirect there. Job URLs join HTTP_URL and CGI_PGM with one slash (NEA keeps its form). Co-Authored-By: Claude Opus 5.5 --- MIGRATING.md | 25 +++--- TAP/compat.py | 15 +++- TAP/tap.py | 143 +++++++++++++++++++++++++++------- tests/conftest.py | 3 + tests/test_compat_http.py | 15 ++-- tests/test_compat_units.py | 2 - tests/test_uws_conformance.py | 142 +++++++++++++++++++++++++++++++++ 7 files changed, 293 insertions(+), 52 deletions(-) create mode 100644 tests/test_uws_conformance.py diff --git a/MIGRATING.md b/MIGRATING.md index 6c93ad3..b673bf5 100644 --- a/MIGRATING.md +++ b/MIGRATING.md @@ -17,7 +17,7 @@ line a day to stderr (the web server's error log): | `nea-errors` | errors are VOTable documents; a table not in TAP_SCHEMA returns 400 | plain text; 403 | | `nea-tables` | `/tables` uses VODataService `` elements | per-column ``, ``, ``, ``, `` | | `nea-votable` | VOTable results are `application/xml`, non-char FIELDs carry no DESCRIPTION (units are still written) | `text/xml`, with descriptions on every FIELD | -| `nea-uws` | job documents use `executionDuration`, `application/xml`, no `Z` on `destruction`; the 303 that starts a job (`PHASE=RUN`) carries no `Content-Type`, `Content-Length` or `Connection` lines, because the response ends when the CGI exits; `/async//executionDuration` (camelCase) is accepted as well as `/executionduration` | `executionduration`, `text/xml`, `Z`; the 303 carries `Content-Type: text/plain`, `Content-Length` and `Connection: close`; only `/executionduration` is a valid key | +| `nea-uws` | job documents are `application/xml`, no `Z` on `destruction`; the 303 that starts a job (`PHASE=RUN`) carries no `Content-Type`, `Content-Length` or `Connection` lines, because the response ends when the CGI exits; `/async//executionDuration` (camelCase) is accepted as well as `/executionduration`; job URLs join `HTTP_URL` and `CGI_PGM` with `/` as written, so `CGI_PGM=/TAP` gives `//TAP` | `text/xml`, `Z`; the 303 carries `Content-Type: text/plain`, `Content-Length` and `Connection: close`; only `/executionduration` is a valid key; job URLs have one `/` between `HTTP_URL` and `CGI_PGM` | ## Moving to the 3.x layout @@ -48,22 +48,27 @@ Compatibility mode reproduces 1.2's responses, with these exceptions: - Errors raised before `TAP.conf` is read (a missing or broken `TAP.conf`, a malformed request path) use the 3.x error format, since the mode isn't known yet. -- `/availability` and `/capabilities` create a workspace directory per - request, so workspace cleanup must cover them. - The async job document escapes quotes in the query text as `'`; it parses to the same XML. -- A job's `status.xml` written under one mode is read correctly under the - other (either spelling of the duration element), so switching `nea-uws` off - doesn't break jobs in flight. +- Both modes write the job document's duration as `executionDuration` (the + UWS name) and read either spelling, so jobs written by an earlier release + or under the other mode read correctly. +- Both modes follow UWS where 1.2 did not: a job can be deleted + (`POST ACTION=DELETE` or HTTP `DELETE`, answered with 303 to the job list), + an unknown job id is 404, `GET` on the job list returns an empty + `` instead of creating a job, and an unset job value (such as + `/destruction` before the job runs) is empty text instead of `None`. `COMPAT` is read only from `[WEB]`; placing it in any other section, including a 1.x `[webserver]`, is a configuration error. ## Release plan -- **3.1.0**: compatibility mode, the 1.x reader, this guide. -- **3.x**: where the shared behavior is to change (for example - `executionDuration` casing and VOTable errors, which IVOA standards - require), the release notes say so before any default changes. +- **3.1.0**: compatibility mode, the 1.x reader, this guide. The default + job document now spells the duration `executionDuration`, as UWS + requires, which retires that part of `nea-uws`. +- **3.x**: where the shared behavior is to change (for example VOTable + errors, which IVOA standards require), the release notes say so before + any default changes. - **4.0.0**: compatibility mode, the `COMPAT` key and the 1.x reader are removed; a 1.x `TAP.conf` then fails at startup with a pointer here. diff --git a/TAP/compat.py b/TAP/compat.py index 965f3f6..96e83ea 100644 --- a/TAP/compat.py +++ b/TAP/compat.py @@ -169,9 +169,15 @@ def field_description(active, desc, coltype): # --- nea-uws ---------------------------------------------------------------- -def duration_element(active): - """UWS names it executionDuration; the shared line writes it lowercase.""" - return 'executionDuration' if 'nea-uws' in active else 'executionduration' +def service_url(active, httpurl, cgipgm): + """The service URL that job URLs extend. + + NEA's carry a double slash (HTTP_URL + '/' + CGI_PGM=/TAP); the shared + line joins the parts with one. + """ + if 'nea-uws' in active: + return httpurl + '/' + cgipgm + return httpurl.rstrip('/') + '/' + cgipgm.strip('/') def uws_key(job, key): @@ -199,7 +205,8 @@ def duration_value(job): """The duration in a parsed status.xml, whichever spelling wrote it. A job written under one mode may be read under the other (jobs live up - to four days), so reading tolerates both; writing uses duration_element. + to four days, and 3.1.0 and earlier wrote it lowercase), so reading + tolerates both; writing uses the UWS spelling. """ return job.get('uws:executionDuration', job.get('uws:executionduration')) diff --git a/TAP/tap.py b/TAP/tap.py index c77b74a..b262862 100755 --- a/TAP/tap.py +++ b/TAP/tap.py @@ -9,6 +9,8 @@ import logging import math import os +import re +import shutil import signal import sys import tempfile @@ -170,7 +172,13 @@ def __run__(self, **kwargs): # { tap.init() # - if('debug' in self.form): + # A request with no form fields (an HTTP DELETE) has no list, and + # FieldStorage raises 'not indexable' on any lookup into it. + self.fields = self.form.keys() if self.form.list is not None else [] + self.request_method = os.environ.get('REQUEST_METHOD', '').upper() + self.action = '' + + if('debug' in self.fields): self.debug = 1 if(self.debug): @@ -227,7 +235,7 @@ def __run__(self, **kwargs): self.uwsheader = '' - for key in self.form: + for key in self.fields: if self.debug: logging.debug(f' key: {key:<15} val: {self.form[key].value:s}') @@ -267,6 +275,9 @@ def __run__(self, **kwargs): if(key.lower() == 'token'): self.token = self.form[key].value + if(key.lower() == 'action'): + self.action = self.form[key].value.strip().upper() + self.nparam = len(self.param) if 'format' in self.param: @@ -526,27 +537,6 @@ def __run__(self, **kwargs): - # Special case: HTTP DELETE. We only need - # the REQUEST_METHOD and PATH_INFO environment - # variable. The first must be DELETE and the - # second is the directory of th TAP job we wish - # to delete. - # - - # if("REQUEST_METHOD" in os.environ): - # self.request_method = os.environ["REQUEST_METHOD"] - # else: - # self.request_method = '' - # - # if self.request_method == 'DELETE': - # - # delete_dir = self.workdir + self.pathinfo - # - # if self.debug: - # logging.debug('') - # logging.debug(f'DELETE: {delete_dir:s}') - - # # Initialize statdict dict # @@ -588,6 +578,15 @@ def __run__(self, **kwargs): #} end VOSI static endpoints # + # + # GET on the job list must not create a job: clients follow the + # redirect that answers a job deletion with a GET here. + # + if ((self.tapcontext == 'async') and (self.getstatus == 0) and + (self.querykey == 0) and (len(self.param['phase']) == 0) and + (self.request_method != 'POST')): + self.__printJobList__() + # # sync or async without input workspace id: make workspace, # otherwise retrieve workspace from getstatus id @@ -656,6 +655,15 @@ def __run__(self, **kwargs): # Retrieve workspace from id # + # + # A job id names a directory that tempfile.mkdtemp made under + # /TAP; anything else ('..', '.') would reach outside + # it, and a deletion removes what the id names. + # + if not re.fullmatch(r'tap_[A-Za-z0-9_]+', self.id): + self.__printError__('votable', 'Job not found: ' + self.id, + errcode='404') + self.workspace = self.id self.userWorkdir = self.workdir + '/TAP/' + self.workspace @@ -663,11 +671,18 @@ def __run__(self, **kwargs): # # check if workspace exists # - isExist = os.path.exists(self.userWorkdir) + isExist = os.path.isdir(self.userWorkdir) if (isExist == 0): - self.msg = 'work directory based on input jobid does not exist.' - self.__printError__('votable', self.msg, errcode='500') + self.__printError__('votable', 'Job not found: ' + self.id, + errcode='404') + + if ((self.tapcontext == 'async') and + (self.statuskey.strip('/') == '') and + ((self.request_method == 'DELETE') or + ((self.request_method == 'POST') and + (self.action == 'DELETE')))): + self.__deleteJob__() # # } end of retrieve workspace @@ -712,7 +727,8 @@ def __run__(self, **kwargs): self.statustbl = 'status.xml' self.statuspath = self.userWorkdir + '/' + self.statustbl - self.statusurl = self.httpurl + '/' + self.cgipgm + \ + self.statusurl = \ + compat.service_url(self.compat, self.httpurl, self.cgipgm) + \ '/' + self.tapcontext + '/' + self.workspace if self.debug: @@ -2284,7 +2300,11 @@ def __getStatus__(self, workdir, workspace, key, param, **kwargs): or (key == 'jobId') or (key == 'runId')): - retval = job[keystr] + # xmltodict reads an empty element as None and a nil one + # as a dict; both are an unset value, which is empty text. + retval = job.get(keystr) + if (retval is None) or isinstance(retval, dict): + retval = '' outstr = retval elif((key == 'ownerId') or (key == 'quote')): @@ -2756,6 +2776,69 @@ def __printSyncResponse__(self, status, msg, resulturl, format, **kwargs): # + def __printJobList__(self, **kwargs): + + # + # UWS job list (GET /async). Jobs carry no owner, so the list + # shows none of them rather than everyone's. + # + + body = ('\n' + '\n') + + sys.stdout.write('HTTP/1.1 200 OK\r\n') + sys.stdout.write(f'Content-Type: {compat.uws_content_type(self.compat)}\r\n') + sys.stdout.write(f'Content-Length: {len(body.encode("utf-8"))}\r\n') + sys.stdout.write('Connection: close\r\n\r\n') + sys.stdout.write(body) + sys.stdout.flush() + sys.exit() + + + def __deleteJob__(self, **kwargs): + + # + # UWS job deletion (POST ACTION=DELETE, or HTTP DELETE): stop the + # job if it is running, remove its workspace, and send the client + # to the job list. The job id was checked against the workspace + # name pattern before userWorkdir was built from it. + # + + try: + with open(self.userWorkdir + '/status.xml', 'r') as fp: + job = xmltodict.parse(fp.read())['uws:job'] + + if (job.get('uws:phase') == 'EXECUTING'): + pid = int(job.get('uws:runId') or 0) + if (pid > 0) and (pid != os.getpid()): + os.kill(pid, signal.SIGKILL) + + except Exception as e: + + # A job whose status can't be read or whose process is gone + # is still deleted. + if self.debug: + logging.debug(f'deleteJob: {str(e):s}') + + try: + shutil.rmtree(self.userWorkdir) + + except OSError as e: + self.__printError__('votable', 'Failed to delete job ' + + self.workspace + ': ' + str(e), errcode='500') + + joblist = compat.service_url(self.compat, self.httpurl, self.cgipgm) + \ + '/async' + + sys.stdout.write('HTTP/1.1 303 See Other\r\n') + sys.stdout.write(f'Location: {joblist}\r\n') + sys.stdout.write('Content-Length: 0\r\n') + sys.stdout.write('Connection: close\r\n\r\n') + sys.stdout.flush() + sys.exit() + + def __printAsyncResponse__(self, statusurl, **kwargs): # @@ -3051,8 +3134,8 @@ def __writeStatusMsg__(self, statuspath, statdict, param, **kwargs): fp.write(f" {statdict['endtime']:s}\n") - el = compat.duration_element(self.compat) - fp.write(f" {statdict['duration']:d}\n") + fp.write(f" {statdict['duration']:d}" + "\n") if (statdict['destruction'] is None): fp.write(' \n') diff --git a/tests/conftest.py b/tests/conftest.py index e2e9b20..10f57cf 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -101,6 +101,9 @@ def do_GET(self): def do_POST(self): self._run_cgi() + def do_DELETE(self): + self._run_cgi() + def _run_cgi(self): # Parse PATH_INFO and QUERY_STRING from self.path. from urllib.parse import urlsplit diff --git a/tests/test_compat_http.py b/tests/test_compat_http.py index 970c29a..29672f5 100644 --- a/tests/test_compat_http.py +++ b/tests/test_compat_http.py @@ -119,7 +119,7 @@ def test_uws_is_neas_form(legacy_tap_server): assert camel.text == dur.text -def test_uws_unchanged_in_default_mode(tap_server): +def test_uws_in_default_mode(tap_server): run, url = _completed_job(tap_server) assert run.headers["Content-Type"] == "text/plain" assert int(run.headers["Content-Length"]) == len(run.content) @@ -127,7 +127,7 @@ def test_uws_unchanged_in_default_mode(tap_server): r = requests.get(url, timeout=30) assert r.headers["Content-Type"] == "text/xml" job = ET.fromstring(r.text) - assert job.find(UWS + "executionduration") is not None + assert job.find(UWS + "executionDuration") is not None # UWS spelling, as of 3.1.0 assert job.find(UWS + "destruction").text.endswith("Z") @@ -140,7 +140,7 @@ def test_camelcase_duration_key_unchanged_in_default_mode(tap_server): def _respell_duration(fixture_root, url, old, new): - """Rewrite one job's status.xml as if the other mode had written it.""" + """Rewrite one job's status.xml as an earlier release would have written it.""" jobid = url.rstrip("/").rsplit("/", 1)[1] paths = list(fixture_root.glob("workdir*/**/%s/status.xml" % jobid)) assert len(paths) == 1, paths @@ -158,15 +158,18 @@ def _read_back(url): assert phase.text.strip() == "COMPLETED", phase.text[:300] -def test_status_written_by_default_mode_reads_in_compat(legacy_tap_server, fixture_root): +# Jobs live four days, so a job written lowercase by an earlier release can +# be read after an upgrade, in either mode. + +def test_lowercase_status_reads_in_compat(legacy_tap_server, fixture_root): _, url = _completed_job(legacy_tap_server) _respell_duration(fixture_root, url, "executionDuration", "executionduration") _read_back(url) -def test_status_written_by_compat_reads_in_default_mode(tap_server, fixture_root): +def test_lowercase_status_reads_in_default_mode(tap_server, fixture_root): _, url = _completed_job(tap_server) - _respell_duration(fixture_root, url, "executionduration", "executionDuration") + _respell_duration(fixture_root, url, "executionDuration", "executionduration") _read_back(url) diff --git a/tests/test_compat_units.py b/tests/test_compat_units.py index 7059818..a7a94c8 100644 --- a/tests/test_compat_units.py +++ b/tests/test_compat_units.py @@ -34,8 +34,6 @@ def test_field_description_blanks_only_non_char_columns(): def test_uws_functions(): - assert compat.duration_element(compat.ALL) == 'executionDuration' - assert compat.duration_element(compat.NONE) == 'executionduration' assert compat.uws_key({'uws:executionDuration': '1'}, 'executionduration') == 'executionDuration' assert compat.uws_key({'uws:executionduration': '1'}, 'executionduration') == 'executionduration' assert compat.uws_key({'uws:executionDuration': '1'}, 'phase') == 'phase' diff --git a/tests/test_uws_conformance.py b/tests/test_uws_conformance.py new file mode 100644 index 0000000..e8b3fda --- /dev/null +++ b/tests/test_uws_conformance.py @@ -0,0 +1,142 @@ +"""UWS behaviors taplint checks (E-UWS-DENO, -HTDE, -JDED, -JDDE) and what +deleting a job needs to be safe.""" +from __future__ import annotations + +import http.client +import os +import xml.etree.ElementTree as ET +from pathlib import Path +from urllib.parse import urlsplit + +import pytest +import requests +from _helpers import tap_async_url + +from TAP import compat + +UWS = "{http://www.ivoa.net/xml/UWS/v1.0}" +QUERY = "SELECT TOP 2 table_name FROM TAP_SCHEMA.tables" + + +def _submit(server): + r = requests.post(tap_async_url(server), data={"query": QUERY, "lang": "ADQL", "format": "csv"}, + allow_redirects=False, timeout=60) + assert r.status_code == 303, r.text + return r.headers["Location"] + + +def _job(url): + r = requests.get(url, timeout=60) + assert r.status_code == 200, r.text + return ET.fromstring(r.content) + + +def _text(url, key): + r = requests.get(f"{url}/{key}", timeout=60) + assert r.status_code == 200, r.text + return r.text.strip() + + +def _jobs_dir(fixture_root: Path) -> Path: + return fixture_root / "workdir" / "TAP" + + +@pytest.mark.parametrize("phase", ["PENDING", "COMPLETED"]) +def test_job_document_agrees_with_its_resources(tap_server, phase): + """JDED, JDDE: the job document and /executionduration, /destruction match.""" + url = _submit(tap_server) + if phase == "COMPLETED": + requests.post(url + "/phase", data={"PHASE": "RUN"}, allow_redirects=False, timeout=60) + for _ in range(60): + if _text(url, "phase") == "COMPLETED": + break + job = _job(url) + duration = job.find(UWS + "executionDuration") + assert duration is not None, "UWS names the element executionDuration" + assert (duration.text or "") == _text(url, "executionduration") + assert (job.find(UWS + "destruction").text or "") == _text(url, "destruction") + + +@pytest.mark.parametrize("how", ["POST ACTION=DELETE", "HTTP DELETE"]) +def test_deleted_job_is_gone(tap_server, fixture_root, how): + """DENO, HTDE: deleting answers 303 to the job list; the job is then 404.""" + url = _submit(tap_server) + jobid = url.rstrip("/").rsplit("/", 1)[1] + assert (_jobs_dir(fixture_root) / jobid).is_dir() + if how == "HTTP DELETE": + r = requests.delete(url, allow_redirects=False, timeout=60) + else: + r = requests.post(url, data={"ACTION": "DELETE"}, allow_redirects=False, timeout=60) + assert r.status_code == 303, r.text + assert r.headers["Location"].rstrip("/").endswith("/async") + assert requests.get(url, timeout=60).status_code == 404 + assert not (_jobs_dir(fixture_root) / jobid).exists() + + +def test_completed_job_can_be_deleted(tap_server): + url = _submit(tap_server) + requests.post(url + "/phase", data={"PHASE": "RUN"}, allow_redirects=False, timeout=60) + for _ in range(60): + if _text(url, "phase") == "COMPLETED": + break + r = requests.delete(url, allow_redirects=False, timeout=60) + assert r.status_code == 303, r.text + assert requests.get(url, timeout=60).status_code == 404 + + +def test_unknown_job_is_404(tap_server): + url = tap_async_url(tap_server) + "/tap_doesnotexist" + assert requests.get(url, timeout=60).status_code == 404 + assert requests.delete(url, allow_redirects=False, timeout=60).status_code == 404 + + +def _raw_status(server, method, path, body=b""): + """Send path as is: requests would drop '.' and '..' segments first.""" + parts = urlsplit(server) + conn = http.client.HTTPConnection(parts.hostname, parts.port, timeout=60) + headers = {"Content-Type": "application/x-www-form-urlencoded"} if body else {} + conn.request(method, path, body=body, headers=headers) + status = conn.getresponse().status + conn.close() + return status + + +@pytest.mark.parametrize("jobid", ["..", ".", "TAP", "tap_x.y", "notajob"]) +def test_job_ids_outside_the_job_namespace_are_404(tap_server, fixture_root, jobid): + """A job id names a directory under /TAP; nothing else may be reached.""" + before = sorted(os.listdir(fixture_root / "workdir")) + path = urlsplit(tap_async_url(tap_server)).path + "/" + jobid + assert _raw_status(tap_server, "DELETE", path) == 404 + assert _raw_status(tap_server, "POST", path, b"ACTION=DELETE") == 404 + assert _raw_status(tap_server, "GET", path) == 404 + assert sorted(os.listdir(fixture_root / "workdir")) == before + assert _jobs_dir(fixture_root).is_dir() + + +def test_job_list_does_not_create_a_job(tap_server, fixture_root): + """Clients follow the delete redirect with a GET on the job list.""" + before = set(os.listdir(_jobs_dir(fixture_root))) + r = requests.get(tap_async_url(tap_server), allow_redirects=False, timeout=60) + assert r.status_code == 200, r.text + assert ET.fromstring(r.content).tag == UWS + "jobs" + assert set(os.listdir(_jobs_dir(fixture_root))) == before + + +@pytest.mark.parametrize("httpurl, cgipgm", [ + ("https://host.org", "/TAP"), ("https://host.org/", "TAP"), ("https://host.org", "TAP"), + ("https://host.org/", "/TAP/"), +]) +def test_job_urls_have_one_slash_between_parts(httpurl, cgipgm): + assert compat.service_url(compat.NONE, httpurl, cgipgm) == "https://host.org/TAP" + + +def test_nea_keeps_its_job_url_form(): + # NEA's job URLs carry the double slash today (HTTP_URL + '/' + CGI_PGM=/TAP). + assert compat.service_url(compat.ALL, "https://host.org", "/TAP") == "https://host.org//TAP" + + +def test_delete_works_in_compat_mode(legacy_tap_server, fixture_root): + url = _submit(legacy_tap_server) + r = requests.delete(url, allow_redirects=False, timeout=60) + assert r.status_code == 303, r.text + assert requests.get(url, timeout=60).status_code == 404 From 31197ab34ef9f2e0b3bf719cf45436e6964feea9 Mon Sep 17 00:00:00 2001 From: BJ Fulton Date: Tue, 27 Oct 2026 08:45:34 -0600 Subject: [PATCH 29/31] Size Oracle DATE columns like DATETIME in IPAC-table output cx_Oracle 8 reports DATE columns as DB_TYPE_DATE, which matched only the 'DATE' check, so the IPAC writer gave them a fixed 14-character width. Oracle DATE values include a time of day (YYYY-MM-DD HH:MM:SS), so every IPAC-format query returning a DATE column failed with "overflows column width [14 char]". cx_Oracle 7 reported these as DATETIME (30 characters); restore that. CSV and VOTable output were unaffected. Co-Authored-By: Claude Opus 5.5 --- TAP/writeresult.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/TAP/writeresult.py b/TAP/writeresult.py index 1d499f1..09aa869 100644 --- a/TAP/writeresult.py +++ b/TAP/writeresult.py @@ -359,6 +359,12 @@ def __init__(self, cursor, workdir, dd, **kwargs): if(ind != -1): dbdatatype = 'TIMESTAMP' + # cx_Oracle 8+ reports DATE columns as DB_TYPE_DATE, which only + # matches 'DATE' above. Oracle DATE values carry a time of day, + # so treat them as DATETIME, as cx_Oracle 7 did. + if dbdatatype == 'DATE': + dbdatatype = 'DATETIME' + ind = dbdatatypestr.find("NUMBER") if(ind != -1): dbdatatype = 'NUMBER' From f7372a6d682de125ac961d8e8418c580b48eab0e Mon Sep 17 00:00:00 2001 From: BJ Fulton Date: Tue, 27 Oct 2026 08:51:23 -0600 Subject: [PATCH 30/31] Port the 3.0.2 proprietary-access fixes to develop The 3.0.2 hotfix (fcab318) was made on main from v3.0.1 and never reached develop, so 3.1 builds lacked it. Bring it over: - group the whole user WHERE predicate before the proprietary constraint is applied, and reject query shapes the filter cannot rewrite safely (aliases, joins, subqueries, comments), checking every referenced table - bind cookie user IDs for Oracle and PostgreSQL - stop logging passwords, cookies and request values; only the server environment (TAP_DEBUG=1) can turn on request debugging - add tests/test_propfilter_security.py Conflicts kept develop's form handling (DELETE requests) and logging style. Also drops a password debug line that compat_vositables.py had gained. The tests now give Tap a FieldStorage-like form, since develop checks form.list. Release metadata, HOTFIX-3.0.2.md and the 3.0.1-only conftest stay on main. ruff passes; 238 passed, 2 skipped. Co-Authored-By: Claude Opus 5.5 --- TAP/compat_vositables.py | 3 - TAP/configparam.py | 7 - TAP/propfilter.py | 81 +++++--- TAP/tap.py | 58 +++--- TAP/tapquery.py | 3 - TAP/taputil.py | 1 - TAP/vositables.py | 4 - tests/test_propfilter_security.py | 332 ++++++++++++++++++++++++++++++ 8 files changed, 420 insertions(+), 69 deletions(-) create mode 100644 tests/test_propfilter_security.py diff --git a/TAP/compat_vositables.py b/TAP/compat_vositables.py index 26f20b9..64c57db 100644 --- a/TAP/compat_vositables.py +++ b/TAP/compat_vositables.py @@ -176,9 +176,6 @@ def __init__(self, **kwargs): raise Exception (self.msg) - if self.debug: - logging.debug('') - logging.debug(f'password= {self.password:s}') elif (self.dbms.lower() == 'sqlite3'): diff --git a/TAP/configparam.py b/TAP/configparam.py index 891dfa9..ca079c5 100644 --- a/TAP/configparam.py +++ b/TAP/configparam.py @@ -75,11 +75,6 @@ def __init__(self, path, **kwargs): confobj = configobj.ConfigObj(self.configpath) confobj, self.compat = compat.load_config(confobj) - if self.debug: - logging.debug('') - logging.debug(' confobj:') - logging.debug(' %s', confobj) - if self.debug: logging.debug('') logging.debug('ConfigObj instantiated successfully') @@ -564,7 +559,6 @@ def __init__(self, path, **kwargs): logging.debug('self.hostname = %s', self.hostname) logging.debug('self.userid = %s', self.userid) logging.debug('self.username = %s', self.username) - logging.debug('self.password = %s', self.password) logging.debug('self.db = %s', self.db) logging.debug('self.database = %s', self.database) logging.debug('self.tap_schema = %s', self.tap_schema) @@ -664,7 +658,6 @@ def __init__(self, path, **kwargs): logging.debug(' hostname = ' + str(self.hostname)) logging.debug(' userid = ' + str(self.userid)) logging.debug(' username = ' + str(self.username)) - logging.debug(' password = ' + str(self.password)) logging.debug(' db = ' + str(self.db)) logging.debug(' database = ' + str(self.database)) logging.debug(' tap_schema = ' + str(self.tap_schema)) diff --git a/TAP/propfilter.py b/TAP/propfilter.py index 3a38fc2..03562ce 100644 --- a/TAP/propfilter.py +++ b/TAP/propfilter.py @@ -5,6 +5,7 @@ import logging import os +import re import time from TAP.datadictionary import dataDictionary @@ -13,6 +14,39 @@ from TAP.writeresult import writeResult +def _filterable(query, wherestr): + """Conservatively accept only query shapes this filter can rewrite safely.""" + q = ' ' + ' '.join(query.lower().split()) + ' ' + if q.count('select') != 1 or ' from ' not in q: + return False + # Comments could consume constraints appended to the user's expression. + # Reject them even inside literals; this is a deliberately strict hotfix. + if any(marker in query for marker in (';', '--', '/*', '*/')): + return False + if re.search(r'\bjoin\b', q): + return False + from_part = re.split(r'\b(?:where|group by|order by)\b', + q.split(' from ', 1)[1], maxsplit=1)[0].strip() + # The final query reconstructs FROM using only the table name, so aliases + # and other FROM modifiers are unsupported as well as multiple tables. + identifier = r'(?:[a-z_][a-z0-9_$#]*|"[a-z0-9_$#]+")' + if not re.fullmatch(identifier + r'(?:\.' + identifier + r')*', from_part): + return False + depth = 0 + for ch in wherestr: + depth += (ch == '(') - (ch == ')') + if depth < 0: + return False + return depth == 0 and ';' not in wherestr + + +def _require_filterable(query, wherestr=''): + if not _filterable(query, wherestr): + raise Exception('This query is not supported on tables with ' + 'proprietary data: use a single table without ' + 'aliases, joins, subqueries or SQL comments.') + + class propFilter: pid = os.getpid() @@ -226,10 +260,6 @@ def __init__(self, **kwargs): logging.debug( 'userid = [Not shown for security reasons].') logging.debug( 'password = [Not shown for security reasons].') - # Change to the following to temporarily debug login - - # logging.debug(f'userid = {self.userid:s}') - # logging.debug(f'password = {self.password:s}') @@ -399,7 +429,6 @@ def __init__(self, **kwargs): logging.debug('') logging.debug('kwargs:') logging.debug(f' userworkdir = {self.userworkdir:s}') - logging.debug(f' cookiestr = {self.cookiestr:s}') logging.debug(f' usertbl = {self.usertbl:s}') logging.debug(f' accesstbl = {self.accesstbl:s}') logging.debug(f' fileid = {self.fileid:s}') @@ -622,6 +651,8 @@ def __init__(self, **kwargs): self.__parseQuery__(self.query) + _require_filterable(self.query, self.wherestr) + if self.debug: logging.debug('') logging.debug('Returned parseQuery:') @@ -902,7 +933,6 @@ def __init__(self, **kwargs): logging.debug ('') logging.debug ('xxx0') logging.debug (f'userid= {userid:s}') - logging.debug (f'password= {password:s}') logging.debug (f'dbserver= {dbserver:s}') try: @@ -1294,7 +1324,6 @@ def __validateUser__(self, cookiename, cookiestr, propfilter, if self.debug: logging.debug('cookiename = ' + str(cookiename)) - logging.debug('cookiestr = ' + str(cookiestr)) msg = '' ind = cookiestr.find(cookiename) @@ -1330,10 +1359,7 @@ def __validateUser__(self, cookiename, cookiestr, propfilter, if self.debug: logging.debug('') - logging.debug('substr1 = ' + str(substr1)) - logging.debug('arr = ' + str(arr)) logging.debug('userid = ' + str(self.userid)) - logging.debug('encodedpass = ' + str(self.encodedpass)) if(self.userid == 'anon'): self.userid = '' @@ -1349,18 +1375,18 @@ def __validateUser__(self, cookiename, cookiestr, propfilter, if(propfilter == 'koa'): sql = "select passwd from " + usertbl + \ - " where userid='" + self.userid + "'" + " where userid = " + self._bind('userid') elif(propfilter == 'neid'): sql = "select password from " + usertbl + \ - " where userid='" + self.userid + "'" + " where userid = " + self._bind('userid') if self.debug: logging.debug('') logging.debug(f'User lookup sql= {sql:s}') try: - self.__executeSql__(cursor, sql) + self.__executeSql__(cursor, sql, {'userid': self.userid}) except Exception as e: @@ -1396,8 +1422,6 @@ def __validateUser__(self, cookiename, cookiestr, propfilter, if self.debug: logging.debug('') logging.debug('password = [Not shown for security reasons.]') - # Change to following to debug password - # logging.debug(f'password = {password:s}') if(len(password) == 0): @@ -1456,16 +1480,9 @@ def __singleValueResult__(self, cursor, keyword, **kwargs): else: valstr = str(val) - if self.debug: - logging.debug(f'valstr= {valstr:s}') - if(colname.lower() == keyword): keyval = valstr - if self.debug: - logging.debug('') - logging.debug(f'keyval= {keyval:s}') - return(keyval) # @@ -1648,7 +1665,7 @@ def __createTmpAccessiddb__(self, tmp_accessiddbtbl, userid, accessid, sql = "insert into " + tmp_accessiddbtbl + \ "(select lower(" + accessid + ") as " + accessid + \ - " from " + accesstbl + " where userid = '" + userid + "')" + " from " + accesstbl + " where userid = " + self._bind('userid') + ")" if self.debug: logging.debug('') @@ -1656,7 +1673,7 @@ def __createTmpAccessiddb__(self, tmp_accessiddbtbl, userid, accessid, cursor = self.conn.cursor() try: - self.__executeSql__(cursor, sql) + self.__executeSql__(cursor, sql, {'userid': userid}) except Exception as e: @@ -1886,8 +1903,9 @@ def __createTmpFileiddb__(self, tmp_fileiddbtbl, fileid, fileid_allowed, selectstr = '' if(len(wherestr) > 0): - selectstr = "select " + fileid + " from " + dbtable + " " + \ - wherestr + " and " + access_constraint + condition = wherestr.strip()[5:].strip() + selectstr = "select " + fileid + " from " + dbtable + \ + " where (" + condition + ") and " + access_constraint else: selectstr = "select " + fileid + " from " + dbtable + \ " where " + access_constraint @@ -2006,14 +2024,21 @@ def __createTmpFileiddb__(self, tmp_fileiddbtbl, fileid, fileid_allowed, # - def __executeSql__(self, cursor, sql, **kwargs): + def _bind(self, name): + return '%(' + name + ')s' if self.dbms.lower() == 'pgsql' else ':' + name + + + def __executeSql__(self, cursor, sql, params=None, **kwargs): # # { # try: - cursor.execute(sql) + if params is None: + cursor.execute(sql) + else: + cursor.execute(sql, params) except Exception as e: diff --git a/TAP/tap.py b/TAP/tap.py index b262862..8eda818 100755 --- a/TAP/tap.py +++ b/TAP/tap.py @@ -23,7 +23,7 @@ from TAP import compat from TAP.configparam import configParam -from TAP.propfilter import propFilter +from TAP.propfilter import _require_filterable, propFilter from TAP.tablenames import TableNames from TAP.tablevalidator import TableValidationError, TableValidator from TAP.tapquery import tapQuery @@ -178,8 +178,8 @@ def __run__(self, **kwargs): self.request_method = os.environ.get('REQUEST_METHOD', '').upper() self.action = '' - if('debug' in self.fields): - self.debug = 1 + # Only the server environment may turn on request debugging. + self.debug = 1 if os.environ.get('TAP_DEBUG') == '1' else 0 if(self.debug): @@ -193,7 +193,7 @@ def __run__(self, **kwargs): logging.debug(f'Enter Tap.init(): pid= {self.pid:d}') # - # Print all environ keys, retrieve async/sync spec from PATH_INFO + # Print operational environ keys, retrieve async/sync spec from PATH_INFO # environ variable # @@ -202,8 +202,9 @@ def __run__(self, **kwargs): logging.debug('Environment parameters:') logging.debug('') - for key in os.environ.keys(): - logging.debug(f' {key:s}: {os.environ[key]:20s}') + for key in ('PATH_INFO', 'REQUEST_METHOD', 'TAP_CONF', 'TAP_DEBUG'): + if key in os.environ: + logging.debug(f' {key:s}: {os.environ[key]:20s}') # @@ -221,7 +222,7 @@ def __run__(self, **kwargs): if self.debug: logging.debug('') - logging.debug('nexsciTAP version 3.0.1\n\n') + logging.debug('nexsciTAP version 3.0.2\n\n') logging.debug('HTTP request keywords:\n') self.lang = 'ADQL' @@ -237,7 +238,7 @@ def __run__(self, **kwargs): for key in self.fields: if self.debug: - logging.debug(f' key: {key:<15} val: {self.form[key].value:s}') + logging.debug(f' key: {key:<15}') if(key.lower() == 'instance'): self.instance = self.form[key].value @@ -438,20 +439,9 @@ def __run__(self, **kwargs): self.cookiestr = os.getenv('HTTP_COOKIE', default='') - if self.debug: - logging.debug('') - logging.debug(f'cookiestr (http) = {self.cookiestr:s}') - if (len(self.cookiestr) == 0): self.cookiestr = self.token - if self.debug: - logging.debug('') - logging.debug(f'cookiestr (token) = {self.cookiestr:s}') - - if self.debug: - logging.debug('') - logging.debug(f'cookiestr (finale) = {self.cookiestr:s}') # # Extract configfile name from TAP_CONF environment variable @@ -1647,6 +1637,31 @@ def __run__(self, **kwargs): # Determine whether to use runQuery or propFilter to execute SQL # + # Check every referenced table before the first-table routing and + # TAP_SCHEMA exemption. A public outer table must not hide a protected + # table referenced by a join or subquery. + info = self.config.connectInfo + schema_key = 'tap_schema_file' if info['dbms'].lower() == 'sqlite3' else 'tap_schema' + metadata_prefix = info.get(schema_key, 'TAP_SCHEMA').lower() + '.' + data_tables = [table for table in tables + if not table.lower().startswith(metadata_prefix)] + policy = self.config.propfilter.lower() + requires_filter = ((policy == 'koa' and bool(data_tables)) or + (policy == 'neid' and any(self.__getDatalevel__(table) != 'l0' + for table in data_tables))) + if requires_filter: + try: + _require_filterable(self.query) + except Exception as e: + self.phase = 'ERROR' + if self.tapcontext == 'async': + self.__writeAsyncError__(str(e), self.statuspath, + self.statdict, self.param) + else: + self.__printError__(self.format, str(e), errcode='400') + return + self.propflag = 1 + if(self.propflag == -1): if((self.config.propfilter.lower() == 'koa') @@ -1700,9 +1715,7 @@ def __run__(self, **kwargs): # Check if dbtable is THE tap_schema table # - ind = self.dbtable.lower().find('tap_schema') - - if(ind != -1): + if not data_tables: self.propflag = 0 if self.debug: @@ -3268,7 +3281,6 @@ def __printVosiTables__ (self, connectInfo, vosipath, **kwargs): logging.debug(f'dbms= {dbms:s}') logging.debug(f'dbserver= {dbserver:s}') logging.debug(f'userid= {userid:s}') - logging.debug(f'password= {password:s}') try: diff --git a/TAP/tapquery.py b/TAP/tapquery.py index 76129b4..43c9173 100644 --- a/TAP/tapquery.py +++ b/TAP/tapquery.py @@ -193,7 +193,6 @@ def __init__(self, **kwargs): if self.debug: logging.debug(f'userid = {self.userid:s}') - logging.debug(f'password = {self.password:s}') logging.debug(f'dbserver = {self.dbserver:s}') @@ -292,7 +291,6 @@ def __init__(self, **kwargs): logging.debug(self.socket) logging.debug(f'db = {self.db:s}') logging.debug(f'userid = {self.userid:s}') - logging.debug(f'password = {self.password:s}') # POSTGRESQL @@ -345,7 +343,6 @@ def __init__(self, **kwargs): logging.debug(f'hostname = {self.hostname:s}') logging.debug(f'database = {self.database:s}') logging.debug(f'username = {self.username:s}') - logging.debug(f'password = {self.password:s}') diff --git a/TAP/taputil.py b/TAP/taputil.py index af4c19f..86c4c82 100644 --- a/TAP/taputil.py +++ b/TAP/taputil.py @@ -147,7 +147,6 @@ def __init__(self, **kwargs): logging.debug('db = ' + str(db)) logging.debug('database = ' + str(database)) logging.debug('userid = ' + str(userid)) - logging.debug('password = ' + str(password)) except Exception as e: raise Exception(e) diff --git a/TAP/vositables.py b/TAP/vositables.py index 2b15001..c6c0702 100644 --- a/TAP/vositables.py +++ b/TAP/vositables.py @@ -168,10 +168,6 @@ def __init__(self, **kwargs): raise Exception (self.msg) - if self.debug: - logging.debug('') - logging.debug(f'password= {self.password:s}') - elif (self.dbms.lower() == 'sqlite3'): if ((self.db is None) or (len(self.db) == 0)): diff --git a/tests/test_propfilter_security.py b/tests/test_propfilter_security.py new file mode 100644 index 0000000..b928503 --- /dev/null +++ b/tests/test_propfilter_security.py @@ -0,0 +1,332 @@ +"""Regression coverage for proprietary row filtering and credential handling.""" +from __future__ import annotations + +import logging +import re +import sqlite3 +import sys +from types import SimpleNamespace + +import pytest + +from TAP import propfilter, vositables +from TAP.configparam import configParam +from TAP.propfilter import propFilter +from TAP.tap import Tap +from TAP.tapquery import tapQuery +from TAP.taputil import tapUtil + + +class _Form(dict): + """Stand-in for cgi.FieldStorage: a key lookup plus the .list Tap checks.""" + + @property + def list(self): + return list(self.values()) + +ORACLE_CONFIG = """[WEB] +TAP_WORKDIR={work}/ +TAP_WORKURL=/workspace/ +HTTP_URL=http://localhost +HTTP_PORT=80 +CGI_PGM=/TAP +[DBMS] +DBMS=oracle +ServerName=testdb +UserID=test +Password=secret +COOKIENAME=tapcookie +PROPFILTER=neid +FILEID=filename +ACCESSID=program +RACOL=ra +DECCOL=dec +[SPTIND] +MODE=HTM +LEVEL=20 +XCOL=x +YCOL=y +ZCOL=z +COLNAME=htm20 +ENCODING=BASE10 +""" + + +@pytest.mark.parametrize("query, where", [ + ("select * from neidl2", ""), + ("SELECT COUNT(*) FROM neidl2 WHERE obstype = 'Sci' OR obstype = 'Cal'", + "WHERE obstype = 'Sci' OR obstype = 'Cal'"), + ("select program, count(*) from neidl2 where (x = 1 or x = 2) group by program", + "where (x = 1 or x = 2)"), +]) +def test_filterable_accepts_single_table_queries(query, where): + assert propfilter._filterable(query, where) + + +@pytest.mark.parametrize("query, where", [ + ("select * from neidl2 join neidl1 on neidl2.x = neidl1.x", ""), + ("select * from neidl2\nLEFT\tJOIN\nneidl1 on neidl2.x = neidl1.x", ""), + ("select * from neidl2, neidl1", ""), + ("select (select x from neidl1) from neidl2", ""), + ("select * from neidl2 where x in (select x from neidl1)", + "where x in (select x from neidl1)"), + ("select * from neidl2 union all select * from neidl1", ""), + ("select * from neidl2 where (x = 1", "where (x = 1"), + ("select * from neidl2 where x = 1)(", "where x = 1)("), + ("select * from neidl2; select * from neidl1", ""), + ("select * from neidl2 where x = 1 -- trailing comment", "where x = 1 -- trailing comment"), + ("select * from neidl2 /* comment */", ""), + ("select * from neidl2 where x = 'select'", "where x = 'select'"), + ("select * from neidl2 a", ""), + ("select * from neidl2 connect by prior x = x", ""), +]) +def test_filterable_refuses_queries_the_filter_cannot_rewrite(query, where): + assert not propfilter._filterable(query, where) + + +class _OracleSQLiteCursor: + """Execute the real filter SQL locally, adapting only Oracle temp-table syntax.""" + + def __init__(self, conn): + self.cursor = conn.cursor() + + def execute(self, sql, params=None): + sql = sql.replace("create global temporary table", "create temporary table") + sql = sql.replace(" on commit preserve rows", "") + sql = re.sub(r"^(insert into \w+)\((select .*)\)$", r"\1 \2", sql) + return self.cursor.execute(sql) if params is None else self.cursor.execute(sql, params) + + def parse(self, sql): + self.cursor.execute("explain " + sql) + + def __getattr__(self, name): + return getattr(self.cursor, name) + + +class _OracleSQLiteConnection: + def __init__(self, conn): + self.conn = conn + + def cursor(self): + return _OracleSQLiteCursor(self.conn) + + +@pytest.mark.parametrize("archive, level", [ + ("koa", ""), ("neid", "l0"), ("neid", "l1"), ("neid", "l2"), ("neid", "eng"), +]) +@pytest.mark.parametrize("authenticated", [False, True]) +@pytest.mark.parametrize("where, public_ids", [ + ("where obstype = 'Sci'", ["public-sci"]), + ("WHERE obstype = 'Sci' OR obstype = 'Sci'", ["public-sci"]), + ("where obstype = 'Sci' OR 1 = 0", ["public-sci"]), + ("where 1 = 0 OR obstype = 'Sci'", ["public-sci"]), + ("where (obstype = 'Sci' OR obstype = 'Cal')", ["public-cal", "public-sci"]), + ("where obstype = 'Sci' OR obstype = 'Cal'", ["public-cal", "public-sci"]), + ("", ["public-cal", "public-sci"]), +]) +def test_allowed_files_apply_access_rules_to_every_or_branch(tmp_path, archive, level, + authenticated, where, public_ids): + with sqlite3.connect(":memory:") as conn: + # Zero-month intervals let SQLite evaluate the date comparison unchanged. + conn.create_function("add_months", 2, lambda date, months: date if months == 0 else None) + conn.execute("create table observations (filename text, obstype text, program text, " + "date_obs text, obsdate text, propint int, l0propint int, l1propint int, l2propint int)") + conn.executemany("insert into observations values (?, ?, ?, ?, ?, 0, 0, 0, 0)", [ + ("public-sci", "Sci", "public", "2000-01-01", "2000-01-01"), + ("public-cal", "Cal", "public", "2000-01-01", "2000-01-01"), + ("owned-sci", "Sci", "owned", "9999-01-01", "9999-01-01"), + ("private-sci", "Sci", "other", "9999-01-01", "9999-01-01"), + ]) + conn.execute("create table allowed_programs (program text)") + conn.execute("insert into allowed_programs values ('owned')") + pf = propFilter.__new__(propFilter) + pf.conn = _OracleSQLiteConnection(conn) + pf.dbms, pf.propfilter, pf.datalevel = "oracle", archive, level + pf.userid = "authorized-user" if authenticated else "" + pf.arraysize = 1000 + pf.userworkdir = str(tmp_path) + pf.__createTmpFileiddb__("allowed_files", "filename", "filename_allowed", + "observations", where, "program", "allowed_programs") + ids = [row[0] for row in conn.execute("select filename_allowed from allowed_files order by 1")] + expected = sorted(public_ids + (["owned-sci"] if authenticated else [])) + assert ids == expected + + +class _RecordingCursor: + description = (("password",),) + arraysize = 1000 + rowcount = 1 + + def __init__(self): + self.calls = [] + + def execute(self, sql, params=None): + self.calls.append((sql, params)) + + def fetchmany(self): + return [("test-credential",)] + + +@pytest.mark.parametrize("dbms, placeholder", [("oracle", ":userid"), ("pgsql", "%(userid)s")]) +@pytest.mark.parametrize("archive, column", [("koa", "passwd"), ("neid", "password")]) +def test_cookie_user_lookup_binds_userid(dbms, placeholder, archive, column): + cursor = _RecordingCursor() + cursor.description = [(column,)] + pf = propFilter.__new__(propFilter) + pf.dbms = dbms + pf.conn = SimpleNamespace(cursor=lambda: cursor) + pf.__validateUser__("tapcookie", "tapcookie=o'brien|test-credential", archive, "users") + assert pf.status == "ok" + assert cursor.calls == [(f"select {column} from users where userid = {placeholder}", + {"userid": "o'brien"})] + + +@pytest.mark.parametrize("dbms, placeholder", [("oracle", ":userid"), ("pgsql", "%(userid)s")]) +def test_access_list_binds_userid(dbms, placeholder, monkeypatch): + cursor = _RecordingCursor() + pf = propFilter.__new__(propFilter) + pf.dbms = dbms + pf.conn = SimpleNamespace(cursor=lambda: cursor) + monkeypatch.setattr(pf, "__dropDbtbl__", lambda _: None) + pf.__createTmpAccessiddb__("allowed_programs", "o'brien", "program", "access_list") + insert = [call for call in cursor.calls if call[0].startswith("insert ")] + assert insert == [(("insert into allowed_programs(select lower(program) as program " + f"from access_list where userid = {placeholder})"), {"userid": "o'brien"})] + + +def test_debug_authentication_does_not_log_credentials(caplog): + cursor = _RecordingCursor() + pf = propFilter.__new__(propFilter) + pf.dbms, pf.debug = "oracle", 1 + pf.conn = SimpleNamespace(cursor=lambda: cursor) + with caplog.at_level(logging.DEBUG): + pf.__validateUser__("tapcookie", "tapcookie=o'brien|test-credential", "neid", "users") + assert pf.status == "ok" + assert "test-credential" not in caplog.text + + +def test_debug_configuration_does_not_log_database_password(tmp_path, caplog): + conf = tmp_path / "TAP.conf" + conf.write_text(ORACLE_CONFIG.format(work=tmp_path).replace("Password=secret", "Password=db-secret-marker")) + with caplog.at_level(logging.DEBUG): + cp = configParam(str(conf), debug=1) + assert cp.connectInfo["password"] == "db-secret-marker" + assert "db-secret-marker" not in caplog.text + + +@pytest.mark.parametrize("caller", ["query", "utility"]) +def test_database_helpers_do_not_log_passwords(caller, tmp_path, monkeypatch, caplog): + def unavailable_connection(*args, **kwargs): + raise RuntimeError("test database unavailable") + + monkeypatch.setitem(sys.modules, "cx_Oracle", SimpleNamespace(connect=unavailable_connection)) + monkeypatch.setattr(sys, "tracebacklimit", getattr(sys, "tracebacklimit", 1000), raising=False) + conf = tmp_path / "TAP.conf" + conf.write_text(ORACLE_CONFIG.format(work=tmp_path).replace("Password=secret", "Password=helper-secret-marker")) + cp = configParam(str(conf)) + with caplog.at_level(logging.DEBUG), pytest.raises(Exception, match="connect"): + if caller == "query": + tapQuery(query="select * from neidl2", workdir=str(tmp_path), connectInfo=cp.connectInfo, debug=1) + else: + tapUtil(configpath=str(conf), debug=1) + assert "helper-secret-marker" not in caplog.text + + +@pytest.mark.parametrize("operator_debug, request_debug, expected", [ + (None, True, 0), ("0", True, 0), ("1", False, 1), +]) +def test_request_cannot_enable_debug(monkeypatch, tmp_path, capsys, operator_debug, request_debug, expected): + if operator_debug is None: + monkeypatch.delenv("TAP_DEBUG", raising=False) + else: + monkeypatch.setenv("TAP_DEBUG", operator_debug) + monkeypatch.setenv("PATH_INFO", "/sync") + monkeypatch.delenv("TAP_CONF", raising=False) + app = Tap.__new__(Tap) + app.form = _Form({"debug": SimpleNamespace(value="1")} if request_debug else {}) + app.debug, app.param, app.infomsg, app.dbtable = 0, {}, "", "" + app.debugfname = str(tmp_path / "tap.debug") + with pytest.raises(SystemExit): + app.__run__() + capsys.readouterr() + assert app.debug == expected + + +def test_operator_debug_does_not_log_cookies_or_tokens(monkeypatch, tmp_path, caplog, capsys): + monkeypatch.setenv("TAP_DEBUG", "1") + monkeypatch.setenv("PATH_INFO", "/sync") + monkeypatch.setenv("HTTP_COOKIE", "tapcookie=user|cookie-secret-marker") + monkeypatch.delenv("TAP_CONF", raising=False) + app = Tap.__new__(Tap) + app.form = _Form({"token": SimpleNamespace(value="tapcookie=user|token-secret-marker")}) + app.debug, app.param, app.infomsg, app.dbtable = 1, {}, "", "" + app.debugfname = str(tmp_path / "tap.debug") + with caplog.at_level(logging.DEBUG), pytest.raises(SystemExit): + app.__run__() + capsys.readouterr() + assert "cookie-secret-marker" not in caplog.text + assert "token-secret-marker" not in caplog.text + + +@pytest.mark.parametrize("query", [ + "select a.program from data_l1 a join data_l0 b on a.program = b.program", + "select program from data_l1 where program in (select program from data_l0)", + "select a.program from data_l1 a, data_l0 b", + "select a.program from data_l0 a join data_l1 b on a.program = b.program", + "select program from data_l0 where program in (select program from data_l1)", + "select table_name from tap_schema.tables where exists (select program from data_l1)", +]) +def test_unsupported_proprietary_query_returns_400(tmp_path, query, monkeypatch, capsys): + def unavailable_connection(*args, **kwargs): + raise RuntimeError("a rejected query must not connect to the database") + + monkeypatch.setitem(sys.modules, "cx_Oracle", SimpleNamespace(connect=unavailable_connection)) + tap_conf = tmp_path / "TAP.conf" + tap_conf.write_text(ORACLE_CONFIG.format(work=tmp_path)) + monkeypatch.setenv("TAP_CONF", str(tap_conf)) + monkeypatch.setenv("PATH_INFO", "/sync") + monkeypatch.delenv("TAP_DEBUG", raising=False) + monkeypatch.delenv("HTTP_COOKIE", raising=False) + app = Tap.__new__(Tap) + app.form = _Form({key: SimpleNamespace(value=value) for key, value in + {"query": query, "format": "votable"}.items()}) + app.debug, app.propflag, app.param, app.statdict = 0, -1, {}, {} + with pytest.raises(SystemExit): + app.__init__() + output = capsys.readouterr().out + assert output.startswith("HTTP/1.1 400"), output[:500] + assert "proprietary data" in output + assert 'value="ERROR"' in output + + +def test_filter_rejects_unsupported_query_before_creating_temporary_tables(tmp_path, monkeypatch): + config = tmp_path / "TAP.conf" + config.write_text(ORACLE_CONFIG.format(work=tmp_path)) + info = configParam(str(config)).connectInfo + config.unlink() + with sqlite3.connect(":memory:") as conn: + conn.execute("attach database ':memory:' as TAP_SCHEMA") + conn.execute("create table TAP_SCHEMA.tables (table_name text)") + conn.executemany("insert into TAP_SCHEMA.tables values (?)", [("data_l0",), ("data_l1",)]) + for table in ("data_l0", "data_l1"): + conn.execute("create table " + table + " (program text)") + bridge = _OracleSQLiteConnection(conn) + monkeypatch.setitem(sys.modules, "cx_Oracle", SimpleNamespace(connect=lambda *args: bridge)) + with pytest.raises(Exception, match="proprietary data"): + propFilter(connectInfo=info, workdir=str(tmp_path), propfilter="neid", + query="select a.program from data_l1 a join data_l0 b on a.program = b.program", + fileid="filename", accessid="program") + assert not conn.execute("select name from sqlite_temp_master").fetchall() + assert list(tmp_path.iterdir()) == [] + + +def test_vosi_debug_does_not_log_database_password(monkeypatch, tmp_path, caplog): + def unavailable_connection(*args, **kwargs): + raise RuntimeError("test database unavailable") + + monkeypatch.setitem(sys.modules, "cx_Oracle", SimpleNamespace(connect=unavailable_connection)) + with caplog.at_level(logging.DEBUG), pytest.raises(Exception, match="Required connectInfo"): + vositables.vosiTables(dbms="oracle", dbserver="test", userid="test", + password="vosi-secret-marker", debug=1, + outpath=str(tmp_path / "tables.xml")) + assert "vosi-secret-marker" not in caplog.text From 27ccc3a74d46d325de12caf3da9dfb49aa9bdb58 Mon Sep 17 00:00:00 2001 From: BJ Fulton Date: Tue, 27 Oct 2026 09:32:29 -0600 Subject: [PATCH 31/31] CI: run the test matrix on ubuntu-24.04 ubuntu-latest now resolves to Ubuntu 26.04, where actions/setup-python has no Python 3.9 build, so test (py3.9) failed during setup on every PR. Pin the test job to 24.04 to keep 3.9 coverage. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 61c8e0e..70ea5bc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,7 +8,9 @@ on: jobs: test: name: test (py${{ matrix.python-version }}) - runs-on: ubuntu-latest + # Pinned: ubuntu-latest moved to 26.04, which has no Python 3.9 build for + # actions/setup-python, so the py3.9 job failed before running any tests. + runs-on: ubuntu-24.04 strategy: fail-fast: false matrix: