diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index 1c1d1af..0163fce 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -26,9 +26,7 @@ jobs: pip install ruff - name: Lint run: | - ruff check --statistics *.py - ruff check --statistics apps/ + ruff check --statistics . - name: Format run: | - ruff format --check *.py - ruff format --check apps/ + ruff format --check . diff --git a/.ruff.toml b/.ruff.toml new file mode 100644 index 0000000..fedb79a --- /dev/null +++ b/.ruff.toml @@ -0,0 +1,85 @@ +# Exclude a variety of commonly ignored directories. +exclude = [ + ".bzr", + ".direnv", + ".eggs", + ".git", + ".git-rewrite", + ".hg", + ".ipynb_checkpoints", + ".mypy_cache", + ".nox", + ".pants.d", + ".pyenv", + ".pytest_cache", + ".pytype", + ".ruff_cache", + ".svn", + ".tox", + ".venv", + ".vscode", + "__pypackages__", + "_build", + "buck-out", + "build", + "dist", + "node_modules", + "site-packages", + "venv", +] + +# Same as Black. +line-length = 88 +indent-width = 4 + +# Assume Python 3.11 +target-version = "py311" + +[lint] +# Rules can be found at https://docs.astral.sh/ruff/rules +select = ["E4", "E7", "E9", "F", "E1", "E2", "W", "N", "D", "B", "C4", "PD", "PERF"] +ignore = [ + "C408", "C420", "C901", "C416", + "D400", "D401", "D100", "D102", "D103", "D104", "D415", "D419", "D101", + "E731", "D205", + "N812", "N806", "N803", + "PLR0913", + "D420" +] + +# Allow fix for all enabled rules (when `--fix`) is provided. +fixable = ["ALL"] +unfixable = [] + +# Allow unused variables when underscore-prefixed. +dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" + +[format] +# Like Black, use double quotes for strings. +quote-style = "double" + +# Like Black, indent with spaces, rather than tabs. +indent-style = "space" + +# Like Black, respect magic trailing commas. +skip-magic-trailing-comma = false + +# Like Black, automatically detect the appropriate line ending. +line-ending = "auto" + +# Enable auto-formatting of code examples in docstrings. Markdown, +# reStructuredText code/literal blocks and doctests are all supported. +# +# This is currently disabled by default, but it is planned for this +# to be opt-out in the future. +docstring-code-format = false + +# Set the line length limit used when formatting code snippets in +# docstrings. +# +# This only has an effect when the `docstring-code-format` setting is +# enabled. +docstring-code-line-length = "dynamic" + +[lint.pydocstyle] +convention = "numpy" diff --git a/app_lsst.py b/app_lsst.py index a01176e..68b5f2b 100644 --- a/app_lsst.py +++ b/app_lsst.py @@ -13,35 +13,34 @@ # See the License for the specific language governing permissions and # limitations under the License. import os -from flask import Flask, Blueprint + +from flask import Blueprint, Flask from flask_restx import Api -from prometheus_flask_exporter.multiprocess import GunicornPrometheusMetrics -from config_prometheus import child_exit, pre_fork, post_fork from prometheus_client import values from prometheus_client.values import MultiProcessValue +from prometheus_flask_exporter.multiprocess import GunicornPrometheusMetrics from apps import __version__ - -from apps.utils.utils import extract_configuration - -from apps.routes.v1.lsst.sources.api import ns as ns_sources -from apps.routes.v1.lsst.objects.api import ns as ns_objects -from apps.routes.v1.lsst.fp.api import ns as ns_fp +from apps.routes.v1.lsst.blocks.api import ns as ns_blocks from apps.routes.v1.lsst.conesearch.api import ns as ns_conesearch from apps.routes.v1.lsst.cutouts.api import ns as ns_cutouts -from apps.routes.v1.lsst.schema.api import ns as ns_schema -from apps.routes.v1.lsst.sso.api import ns as ns_sso +from apps.routes.v1.lsst.fp.api import ns as ns_fp +from apps.routes.v1.lsst.objects.api import ns as ns_objects from apps.routes.v1.lsst.resolver.api import ns as ns_resolver +from apps.routes.v1.lsst.schema.api import ns as ns_schema from apps.routes.v1.lsst.skymap.api import ns as ns_skymap +from apps.routes.v1.lsst.sources.api import ns as ns_sources +from apps.routes.v1.lsst.sso.api import ns as ns_sso from apps.routes.v1.lsst.statistics.api import ns as ns_stats from apps.routes.v1.lsst.tags.api import ns as ns_tags -from apps.routes.v1.lsst.blocks.api import ns as ns_blocks +from apps.utils.utils import extract_configuration +from config_prometheus import child_exit, post_fork, pre_fork config = extract_configuration("config.yml") def get_worker_id(): - """return stable id for worker""" + """Return stable id for worker""" return os.environ.get("GUNICORN_WORKER_ID", str(os.getpid())) diff --git a/app_ztf.py b/app_ztf.py index 61e8b96..a926b84 100644 --- a/app_ztf.py +++ b/app_ztf.py @@ -13,39 +13,38 @@ # See the License for the specific language governing permissions and # limitations under the License. import os -from flask import Flask, Blueprint + +from flask import Blueprint, Flask from flask_restx import Api -from prometheus_flask_exporter.multiprocess import GunicornPrometheusMetrics -from config_prometheus import child_exit, pre_fork, post_fork from prometheus_client import values from prometheus_client.values import MultiProcessValue +from prometheus_flask_exporter.multiprocess import GunicornPrometheusMetrics from apps import __version__ - -from apps.utils.utils import extract_configuration - -from apps.routes.v1.ztf.objects.api import ns as ns_objects -from apps.routes.v1.ztf.cutouts.api import ns as ns_cutouts -from apps.routes.v1.ztf.latests.api import ns as ns_latests +from apps.routes.v1.ztf.anomaly.api import ns as ns_anomaly from apps.routes.v1.ztf.classes.api import ns as ns_classes from apps.routes.v1.ztf.conesearch.api import ns as ns_conesearch -from apps.routes.v1.ztf.sso.api import ns as ns_sso +from apps.routes.v1.ztf.cutouts.api import ns as ns_cutouts +from apps.routes.v1.ztf.latests.api import ns as ns_latests +from apps.routes.v1.ztf.metadata.api import ns as ns_metadata +from apps.routes.v1.ztf.objects.api import ns as ns_objects from apps.routes.v1.ztf.resolver.api import ns as ns_resolver -from apps.routes.v1.ztf.tracklet.api import ns as ns_tracklet from apps.routes.v1.ztf.schema.api import ns as ns_schema from apps.routes.v1.ztf.skymap.api import ns as ns_skymap -from apps.routes.v1.ztf.statistics.api import ns as ns_statistics +from apps.routes.v1.ztf.sso.api import ns as ns_sso +from apps.routes.v1.ztf.ssobulk.api import ns as ns_ssobulk from apps.routes.v1.ztf.ssocand.api import ns as ns_ssocand -from apps.routes.v1.ztf.anomaly.api import ns as ns_anomaly from apps.routes.v1.ztf.ssoft.api import ns as ns_ssoft -from apps.routes.v1.ztf.metadata.api import ns as ns_metadata -from apps.routes.v1.ztf.ssobulk.api import ns as ns_ssobulk +from apps.routes.v1.ztf.statistics.api import ns as ns_statistics +from apps.routes.v1.ztf.tracklet.api import ns as ns_tracklet +from apps.utils.utils import extract_configuration +from config_prometheus import child_exit, post_fork, pre_fork config = extract_configuration("config.yml") def get_worker_id(): - """return stable id for worker""" + """Return stable id for worker""" return os.environ.get("GUNICORN_WORKER_ID", str(os.getpid())) diff --git a/apps/routes/v1/lsst/blocks/api.py b/apps/routes/v1/lsst/blocks/api.py index 2777ce1..3f4ce42 100644 --- a/apps/routes/v1/lsst/blocks/api.py +++ b/apps/routes/v1/lsst/blocks/api.py @@ -25,5 +25,5 @@ class Blocks(Resource): def get(self): """Retrieve block definition""" tags, descriptions = extract_blocks(True) - out = {k: v for k, v in zip(tags, descriptions)} + out = {k: v for k, v in zip(tags, descriptions, strict=True)} return jsonify(out) diff --git a/apps/routes/v1/lsst/blocks/utils.py b/apps/routes/v1/lsst/blocks/utils.py index c25d8ec..2204851 100644 --- a/apps/routes/v1/lsst/blocks/utils.py +++ b/apps/routes/v1/lsst/blocks/utils.py @@ -12,9 +12,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import fink_filters.rubin.blocks as fblocks import importlib +import fink_filters.rubin.blocks as fblocks + def extract_blocks(with_description=False): """Extract user-defined blocks diff --git a/apps/routes/v1/lsst/conesearch/api.py b/apps/routes/v1/lsst/conesearch/api.py index cf95d14..165d071 100644 --- a/apps/routes/v1/lsst/conesearch/api.py +++ b/apps/routes/v1/lsst/conesearch/api.py @@ -15,10 +15,8 @@ from flask import Response, request from flask_restx import Namespace, Resource, fields -from apps.utils.utils import check_args -from apps.utils.utils import send_tabular_data - from apps.routes.v1.lsst.conesearch.utils import run_conesearch +from apps.utils.utils import check_args, send_tabular_data ns = Namespace("api/v1/conesearch", "Get Rubin alerts data based on coordinates") diff --git a/apps/routes/v1/lsst/conesearch/test.py b/apps/routes/v1/lsst/conesearch/test.py index 9363afd..5b6a75b 100644 --- a/apps/routes/v1/lsst/conesearch/test.py +++ b/apps/routes/v1/lsst/conesearch/test.py @@ -12,16 +12,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import requests -import pandas as pd -import numpy as np +import io +import sys +import numpy as np +import pandas as pd +import requests from astropy.coordinates import SkyCoord from astropy.io import votable -import io -import sys - APIURL = sys.argv[1] RA0 = 53.2514 @@ -50,7 +49,7 @@ def conesearch( if columns is not None: payload.update({"columns": columns}) - r = requests.post("{}/api/v1/conesearch".format(APIURL), json=payload) + r = requests.post(f"{APIURL}/api/v1/conesearch", json=payload) assert r.status_code == 200, r.content @@ -145,7 +144,7 @@ def test_bad_radius_conesearch() -> None: "output_format": "json", } - r = requests.post("{}/api/v1/conesearch".format(APIURL), json=payload) + r = requests.post(f"{APIURL}/api/v1/conesearch", json=payload) msg = { "status": "error", @@ -167,7 +166,7 @@ def test_conesearch_with_cols() -> None: assert not pdf.empty, pdf # specified fields, plus mandatory i:ra,i:dec, plus computed v:separation - assert len(pdf.columns) == 4, "I count {} columns".format(len(pdf.columns)) + assert len(pdf.columns) == 4, f"I count {len(pdf.columns)} columns" def test_bad_dates() -> None: @@ -185,7 +184,7 @@ def test_bad_dates() -> None: "output_format": "json", } - r = requests.post("{}/api/v1/conesearch".format(APIURL), json=payload) + r = requests.post(f"{APIURL}/api/v1/conesearch", json=payload) msg = { "status": "error", @@ -218,7 +217,7 @@ def test_coordinates() -> None: ] pdf0 = conesearch(ra=RA0, dec=DEC0, columns="r:diaObjectId") for ra, dec in coords: - pdf = conesearch(ra=RA0, dec=DEC0, columns="r:diaObjectId") + pdf = conesearch(ra=ra, dec=dec, columns="r:diaObjectId") assert pdf.equals(pdf0) @@ -230,7 +229,7 @@ def test_bad_request() -> None: """ payload = {"ra": "kfdlkj", "dec": "lkfdjf", "radius": 5, "output_format": "json"} - r = requests.post("{}/api/v1/conesearch".format(APIURL), json=payload) + r = requests.post(f"{APIURL}/api/v1/conesearch", json=payload) msg = { "status": "error", @@ -262,7 +261,7 @@ def test_various_outputs() -> None: if __name__ == "__main__": """ Execute the test suite """ - import sys import doctest + import sys sys.exit(doctest.testmod()[0]) diff --git a/apps/routes/v1/lsst/conesearch/utils.py b/apps/routes/v1/lsst/conesearch/utils.py index dcafbf6..04c5788 100644 --- a/apps/routes/v1/lsst/conesearch/utils.py +++ b/apps/routes/v1/lsst/conesearch/utils.py @@ -12,23 +12,19 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from flask import Response - -import pandas as pd - -from numpy import pi as nppi -from healpy import query_disc, ang2vec - import astropy.units as u -from astropy.time import Time +import pandas as pd from astropy.coordinates import SkyCoord +from astropy.time import Time +from flask import Response +from healpy import ang2vec, query_disc +from line_profiler import profile +from numpy import pi as nppi from apps.utils.client import connect_to_hbase_table from apps.utils.decoding import format_lsst_hbase_output from apps.utils.utils import isoify_time -from line_profiler import profile - @profile def run_conesearch(payload: dict) -> pd.DataFrame: diff --git a/apps/routes/v1/lsst/cutouts/api.py b/apps/routes/v1/lsst/cutouts/api.py index 12d3c16..dadb74b 100644 --- a/apps/routes/v1/lsst/cutouts/api.py +++ b/apps/routes/v1/lsst/cutouts/api.py @@ -15,9 +15,8 @@ from flask import Response, request from flask_restx import Namespace, Resource, fields -from apps.utils.utils import check_args - from apps.routes.v1.lsst.cutouts.utils import format_and_send_cutout +from apps.utils.utils import check_args ns = Namespace("api/v1/cutouts", "Get cutout alert data based on Rubin diaSourceId") diff --git a/apps/routes/v1/lsst/cutouts/profiling.py b/apps/routes/v1/lsst/cutouts/profiling.py index 5cf1137..25d765d 100644 --- a/apps/routes/v1/lsst/cutouts/profiling.py +++ b/apps/routes/v1/lsst/cutouts/profiling.py @@ -14,9 +14,10 @@ # limitations under the License. """Call format_and_send_cutout""" -from apps.routes.v1.lsst.cutouts.utils import format_and_send_cutout from flask import Flask +from apps.routes.v1.lsst.cutouts.utils import format_and_send_cutout + app = Flask("Profile cutouts") payload = { diff --git a/apps/routes/v1/lsst/cutouts/test.py b/apps/routes/v1/lsst/cutouts/test.py index fb2b951..b5a99fd 100644 --- a/apps/routes/v1/lsst/cutouts/test.py +++ b/apps/routes/v1/lsst/cutouts/test.py @@ -12,16 +12,14 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import requests -import numpy as np +import io +import sys +import numpy as np +import requests from astropy.io import fits - from PIL import Image -import io -import sys - APIURL = sys.argv[1] @@ -54,7 +52,7 @@ def cutouttest( if convolution_kernel is not None: payload.update({"convolution_kernel": convolution_kernel}) - r = requests.post("{}/api/v1/cutouts".format(APIURL), json=payload) + r = requests.post(f"{APIURL}/api/v1/cutouts", json=payload) assert r.status_code == 200, r.content @@ -64,7 +62,7 @@ def cutouttest( elif output_format == "FITS": data = fits.open(io.BytesIO(r.content), ignore_missing_simple=True) elif output_format == "array": - data = r.json()["b:cutout{}".format(kind)] + data = r.json()[f"b:cutout{kind}"] return data @@ -190,7 +188,7 @@ def test_convolution_kernel_cutout() -> None: if __name__ == "__main__": """ Execute the test suite """ - import sys import doctest + import sys sys.exit(doctest.testmod()[0]) diff --git a/apps/routes/v1/lsst/cutouts/utils.py b/apps/routes/v1/lsst/cutouts/utils.py index efe6726..4112a06 100644 --- a/apps/routes/v1/lsst/cutouts/utils.py +++ b/apps/routes/v1/lsst/cutouts/utils.py @@ -12,23 +12,21 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from flask import send_file, jsonify, Response - import io import json -import requests import numpy as np +import requests +from flask import Response, jsonify, send_file +from line_profiler import profile from matplotlib import cm from PIL import Image from apps.utils.client import connect_to_hbase_table -from apps.utils.plotting import sigmoid_normalizer, legacy_normalizer, convolve from apps.utils.decoding import format_hbase_output +from apps.utils.plotting import convolve, legacy_normalizer, sigmoid_normalizer from apps.utils.utils import extract_configuration -from line_profiler import profile - @profile def format_and_send_cutout(payload: dict): @@ -165,7 +163,7 @@ def format_and_send_cutout(payload: dict): if "colormap" in payload: colormap = getattr(cm, payload["colormap"]) else: - colormap = lambda x: x # noqa: E731 + colormap = lambda x: x array = np.uint8(colormap(array) * 255) # Convert to PNG @@ -198,16 +196,12 @@ def request_cutout(json_payload, output_format, cutout_api_url): """ if output_format == "FITS": json_payload.update({"return_type": "FITS"}) - r0 = requests.post( - "{}/api/v1/cutouts".format(cutout_api_url), json=json_payload - ) + r0 = requests.post(f"{cutout_api_url}/api/v1/cutouts", json=json_payload) # FIXME: raise of error cutout = io.BytesIO(r0.content) elif output_format in ["PNG", "array"]: json_payload.update({"return_type": "array"}) - r0 = requests.post( - "{}/api/v1/cutouts".format(cutout_api_url), json=json_payload - ) + r0 = requests.post(f"{cutout_api_url}/api/v1/cutouts", json=json_payload) cutout = json.loads(r0.content) # FIXME: raise for error return cutout diff --git a/apps/routes/v1/lsst/fp/api.py b/apps/routes/v1/lsst/fp/api.py index 2a00a2e..403ed85 100644 --- a/apps/routes/v1/lsst/fp/api.py +++ b/apps/routes/v1/lsst/fp/api.py @@ -15,10 +15,8 @@ from flask import Response, request from flask_restx import Namespace, Resource, fields -from apps.utils.utils import check_args -from apps.utils.utils import send_tabular_data - from apps.routes.v1.lsst.fp.utils import extract_fp_data +from apps.utils.utils import check_args, send_tabular_data ns = Namespace("api/v1/fp", "Get forced photometry data based on Rubin diaObjectId") diff --git a/apps/routes/v1/lsst/fp/utils.py b/apps/routes/v1/lsst/fp/utils.py index 7025bb6..422852c 100644 --- a/apps/routes/v1/lsst/fp/utils.py +++ b/apps/routes/v1/lsst/fp/utils.py @@ -13,12 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. import pandas as pd +from line_profiler import profile from apps.utils.client import connect_to_hbase_table from apps.utils.decoding import format_lsst_hbase_output -from line_profiler import profile - @profile def extract_fp_data(payload: dict) -> pd.DataFrame: @@ -50,7 +49,7 @@ def extract_fp_data(payload: dict) -> pd.DataFrame: # single object search salt = payload["diaObjectId"][-3:] key = payload["diaObjectId"] - objectids = ["key:key:{}_{}".format(salt, key)] + objectids = [f"key:key:{salt}_{key}"] if cols == "*": truncated = False diff --git a/apps/routes/v1/lsst/objects/api.py b/apps/routes/v1/lsst/objects/api.py index 8e18f2d..1cb32b5 100644 --- a/apps/routes/v1/lsst/objects/api.py +++ b/apps/routes/v1/lsst/objects/api.py @@ -15,10 +15,8 @@ from flask import Response, request from flask_restx import Namespace, Resource, fields -from apps.utils.utils import check_args -from apps.utils.utils import send_tabular_data - from apps.routes.v1.lsst.objects.utils import extract_object_data +from apps.utils.utils import check_args, send_tabular_data ns = Namespace( "api/v1/objects", "Get diaObject & aggregated data based on Rubin diaObjectId" diff --git a/apps/routes/v1/lsst/objects/test.py b/apps/routes/v1/lsst/objects/test.py index 04d719a..e88887b 100644 --- a/apps/routes/v1/lsst/objects/test.py +++ b/apps/routes/v1/lsst/objects/test.py @@ -12,13 +12,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import requests -import pandas as pd -import numpy as np - import io import sys +import numpy as np +import pandas as pd +import requests + APIURL = sys.argv[1] # Implement random name generator @@ -37,7 +37,7 @@ def get_an_object( "output-format": output_format, } - r = requests.post("{}/api/v1/objects".format(APIURL), json=payload) + r = requests.post(f"{APIURL}/api/v1/objects", json=payload) assert r.status_code == 200, r.content @@ -95,7 +95,7 @@ def test_column_selection() -> None: """ pdf = get_an_object(oid=OID, columns="r:nDiaSources,r:firstDiaSourceMjdTai") - assert len(pdf.columns) == 2, "I count {} columns".format(len(pdf.columns)) + assert len(pdf.columns) == 2, f"I count {len(pdf.columns)} columns" def test_bad_request() -> None: @@ -130,15 +130,13 @@ def test_multiple_objects() -> None: n_oids_single += n_oid len_object += len(pdf_) - assert n_oids == n_oids_single, "{} is not equal to {}".format( - n_oids, n_oids_single - ) - assert len_object == len(pdf), "{} is not equal to {}".format(len_object, len(pdf)) + assert n_oids == n_oids_single, f"{n_oids} is not equal to {n_oids_single}" + assert len_object == len(pdf), f"{len_object} is not equal to {len(pdf)}" if __name__ == "__main__": """ Execute the test suite """ - import sys import doctest + import sys sys.exit(doctest.testmod()[0]) diff --git a/apps/routes/v1/lsst/objects/utils.py b/apps/routes/v1/lsst/objects/utils.py index 66a1f99..bdd1eb5 100644 --- a/apps/routes/v1/lsst/objects/utils.py +++ b/apps/routes/v1/lsst/objects/utils.py @@ -13,12 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. import pandas as pd +from line_profiler import profile from apps.utils.client import connect_to_hbase_table from apps.utils.decoding import format_lsst_hbase_output -from line_profiler import profile - @profile def extract_object_data(payload: dict) -> pd.DataFrame: diff --git a/apps/routes/v1/lsst/schema/api.py b/apps/routes/v1/lsst/schema/api.py index 445d4db..0235659 100644 --- a/apps/routes/v1/lsst/schema/api.py +++ b/apps/routes/v1/lsst/schema/api.py @@ -15,9 +15,8 @@ from flask import Response, request from flask_restx import Namespace, Resource, fields -from apps.utils.utils import check_args - from apps.routes.v1.lsst.schema.utils import extract_schema +from apps.utils.utils import check_args ns = Namespace( "api/v1/schema", "Retrieve the data schema for a given endpoint for Fink/Rubin API" diff --git a/apps/routes/v1/lsst/schema/test.py b/apps/routes/v1/lsst/schema/test.py index 351806b..a1f38f2 100644 --- a/apps/routes/v1/lsst/schema/test.py +++ b/apps/routes/v1/lsst/schema/test.py @@ -12,21 +12,19 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import requests -import numpy as np import logging - import sys +import numpy as np +import requests + APIURL = sys.argv[1] _LOG = logging.getLogger(__name__) def get_fields_per_family(): """Get fields exposed in /api/v1/schema""" - r = requests.post( - "{}/api/v1/schema".format(APIURL), json={"endpoint": "/api/v1/sources"} - ) + r = requests.post(f"{APIURL}/api/v1/schema", json={"endpoint": "/api/v1/sources"}) assert r.status_code == 200, r.content @@ -83,10 +81,8 @@ def check_schema_endpoint(): for endpoint, payload in endpoints.items(): _LOG.warning(endpoint) - r_schema = requests.post( - "{}/api/v1/schema".format(APIURL), json={"endpoint": endpoint} - ) - r_data = requests.post("{}{}".format(APIURL, endpoint), json=payload) + r_schema = requests.post(f"{APIURL}/api/v1/schema", json={"endpoint": endpoint}) + r_data = requests.post(f"{APIURL}{endpoint}", json=payload) if r_data.status_code != 200: _LOG.error(r_data.content) @@ -97,11 +93,7 @@ def check_schema_endpoint(): schema_fields = get_fields_from_schema(r_schema.json()) if len(r_data.json()) == 0: - _LOG.error( - "Data is empty for endpoint {} with payload {}".format( - endpoint, payload - ) - ) + _LOG.error(f"Data is empty for endpoint {endpoint} with payload {payload}") raise ValueError data_fields = list(r_data.json()[0].keys()) @@ -119,7 +111,7 @@ def check_schema_endpoint(): if __name__ == "__main__": """ Execute the test suite """ - import sys import doctest + import sys sys.exit(doctest.testmod()[0]) diff --git a/apps/routes/v1/lsst/schema/utils.py b/apps/routes/v1/lsst/schema/utils.py index 5b109bb..209c887 100644 --- a/apps/routes/v1/lsst/schema/utils.py +++ b/apps/routes/v1/lsst/schema/utils.py @@ -12,11 +12,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from flask import Response import json -import requests import logging +import requests +from flask import Response from line_profiler import profile _LOG = logging.getLogger(__name__) @@ -73,7 +73,7 @@ def extract_schema(payload: dict) -> Response: r = requests.get( "https://raw.githubusercontent.com/lsst/alert_packet/refs/heads/main/python/lsst/alert/packet/schema/latest.txt" ) - version = "{}".format(r.json()) + version = f"{r.json()}" major_version, minor_version = [int(i) for i in version.split(".")] else: major_version = payload["major_version"] @@ -82,9 +82,7 @@ def extract_schema(payload: dict) -> Response: base_url = "https://raw.githubusercontent.com/lsst/alert_packet/refs/heads/main/python/lsst/alert/packet/schema" r_root = requests.get( - "{}/{}/{}/lsst.v{}_{}.alert.avsc".format( - base_url, major_version, minor_version, major_version, minor_version - ) + f"{base_url}/{major_version}/{minor_version}/lsst.v{major_version}_{minor_version}.alert.avsc" ) root_schema = r_root.json() @@ -97,37 +95,27 @@ def extract_schema(payload: dict) -> Response: # Other fields r_diaSource = requests.get( - "{}/{}/{}/lsst.v{}_{}.diaSource.avsc".format( - base_url, major_version, minor_version, major_version, minor_version - ) + f"{base_url}/{major_version}/{minor_version}/lsst.v{major_version}_{minor_version}.diaSource.avsc" ) diaSource_schema = r_diaSource.json()["fields"] r_diaForcedSource = requests.get( - "{}/{}/{}/lsst.v{}_{}.diaForcedSource.avsc".format( - base_url, major_version, minor_version, major_version, minor_version - ) + f"{base_url}/{major_version}/{minor_version}/lsst.v{major_version}_{minor_version}.diaForcedSource.avsc" ) forcedDiaSource_schema = r_diaForcedSource.json()["fields"] r_diaObject = requests.get( - "{}/{}/{}/lsst.v{}_{}.diaObject.avsc".format( - base_url, major_version, minor_version, major_version, minor_version - ) + f"{base_url}/{major_version}/{minor_version}/lsst.v{major_version}_{minor_version}.diaObject.avsc" ) diaObject_schema = r_diaObject.json()["fields"] r_ssSource = requests.get( - "{}/{}/{}/lsst.v{}_{}.ssSource.avsc".format( - base_url, major_version, minor_version, major_version, minor_version - ) + f"{base_url}/{major_version}/{minor_version}/lsst.v{major_version}_{minor_version}.ssSource.avsc" ) ssSource_schema = r_ssSource.json()["fields"] r_mpc_orbits = requests.get( - "{}/{}/{}/lsst.v{}_{}.mpc_orbits.avsc".format( - base_url, major_version, minor_version, major_version, minor_version - ) + f"{base_url}/{major_version}/{minor_version}/lsst.v{major_version}_{minor_version}.mpc_orbits.avsc" ) mpc_orbits_schema = r_mpc_orbits.json()["fields"] diff --git a/apps/routes/v1/lsst/skymap/api.py b/apps/routes/v1/lsst/skymap/api.py index 4fb980c..b34348f 100644 --- a/apps/routes/v1/lsst/skymap/api.py +++ b/apps/routes/v1/lsst/skymap/api.py @@ -15,10 +15,8 @@ from flask import Response, request from flask_restx import Namespace, Resource, fields -from apps.utils.utils import check_args -from apps.utils.utils import send_tabular_data - from apps.routes.v1.lsst.skymap.utils import search_in_skymap +from apps.utils.utils import check_args, send_tabular_data ns = Namespace("api/v1/skymap", "Return Fink/LSST alerts within a GW skymap") diff --git a/apps/routes/v1/lsst/skymap/profiling.py b/apps/routes/v1/lsst/skymap/profiling.py index cb0fce2..059f954 100644 --- a/apps/routes/v1/lsst/skymap/profiling.py +++ b/apps/routes/v1/lsst/skymap/profiling.py @@ -19,4 +19,4 @@ payload = {"event_name": "S251112cm", "credible_level": 0.2} pdf = search_in_skymap(payload) -print("{} objects".format(len(pdf))) +print(f"{len(pdf)} objects") diff --git a/apps/routes/v1/lsst/skymap/test.py b/apps/routes/v1/lsst/skymap/test.py index a058115..66d4ad2 100644 --- a/apps/routes/v1/lsst/skymap/test.py +++ b/apps/routes/v1/lsst/skymap/test.py @@ -12,12 +12,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import requests -import pandas as pd - import io import sys +import pandas as pd +import requests + APIURL = sys.argv[1] @@ -44,7 +44,7 @@ def bayestartest( "output-format": output_format, } - r = requests.post("{}/api/v1/skymap".format(APIURL), json=payload) + r = requests.post(f"{APIURL}/api/v1/skymap", json=payload) assert r.status_code == 200, r.content @@ -87,7 +87,7 @@ def test_name_bayestar() -> None: if __name__ == "__main__": """ Execute the test suite """ - import sys import doctest + import sys sys.exit(doctest.testmod()[0]) diff --git a/apps/routes/v1/lsst/skymap/utils.py b/apps/routes/v1/lsst/skymap/utils.py index d28c98d..7d49b2a 100644 --- a/apps/routes/v1/lsst/skymap/utils.py +++ b/apps/routes/v1/lsst/skymap/utils.py @@ -12,22 +12,20 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import io import gzip -import requests - -from astropy.io import fits -from astropy.time import Time +import io import healpy as hp -import pandas as pd import numpy as np +import pandas as pd +import requests +from astropy.io import fits +from astropy.time import Time +from line_profiler import profile from apps.utils.client import connect_to_hbase_table from apps.utils.decoding import format_hbase_output -from line_profiler import profile - @profile def search_in_skymap(payload: dict) -> pd.DataFrame: diff --git a/apps/routes/v1/lsst/sources/api.py b/apps/routes/v1/lsst/sources/api.py index 966bb73..c5ce7d7 100644 --- a/apps/routes/v1/lsst/sources/api.py +++ b/apps/routes/v1/lsst/sources/api.py @@ -15,10 +15,8 @@ from flask import Response, request from flask_restx import Namespace, Resource, fields -from apps.utils.utils import check_args -from apps.utils.utils import send_tabular_data - from apps.routes.v1.lsst.sources.utils import extract_object_data +from apps.utils.utils import check_args, send_tabular_data ns = Namespace("api/v1/sources", "Get diaSource data based on Rubin diaObjectId") diff --git a/apps/routes/v1/lsst/sources/test.py b/apps/routes/v1/lsst/sources/test.py index ffb49b0..b3222e4 100644 --- a/apps/routes/v1/lsst/sources/test.py +++ b/apps/routes/v1/lsst/sources/test.py @@ -12,13 +12,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import requests -import pandas as pd -import numpy as np - import io import sys +import numpy as np +import pandas as pd +import requests + APIURL = sys.argv[1] # Implement random name generator @@ -41,7 +41,7 @@ def get_an_object( if midpointMjdTai is not None: payload.update({"midpointMjdTai": midpointMjdTai}) - r = requests.post("{}/api/v1/sources".format(APIURL), json=payload) + r = requests.post(f"{APIURL}/api/v1/sources", json=payload) assert r.status_code == 200, r.content @@ -108,7 +108,7 @@ def test_column_selection() -> None: """ pdf = get_an_object(oid=OID, columns="r:midpointMjdTai,r:psfFlux") - assert len(pdf.columns) == 2, "I count {} columns".format(len(pdf.columns)) + assert len(pdf.columns) == 2, f"I count {len(pdf.columns)} columns" def test_formatting() -> None: @@ -172,15 +172,13 @@ def test_multiple_objects() -> None: n_oids_single += n_oid len_object += len(pdf_) - assert n_oids == n_oids_single, "{} is not equal to {}".format( - n_oids, n_oids_single - ) - assert len_object == len(pdf), "{} is not equal to {}".format(len_object, len(pdf)) + assert n_oids == n_oids_single, f"{n_oids} is not equal to {n_oids_single}" + assert len_object == len(pdf), f"{len_object} is not equal to {len(pdf)}" if __name__ == "__main__": """ Execute the test suite """ - import sys import doctest + import sys sys.exit(doctest.testmod()[0]) diff --git a/apps/routes/v1/lsst/sources/utils.py b/apps/routes/v1/lsst/sources/utils.py index 033985e..576bee0 100644 --- a/apps/routes/v1/lsst/sources/utils.py +++ b/apps/routes/v1/lsst/sources/utils.py @@ -13,12 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. import pandas as pd +from line_profiler import profile from apps.utils.client import connect_to_hbase_table from apps.utils.decoding import format_lsst_hbase_output -from line_profiler import profile - @profile def extract_object_data(payload: dict) -> pd.DataFrame: @@ -54,7 +53,7 @@ def extract_object_data(payload: dict) -> pd.DataFrame: key = "{}_{}".format(payload["diaObjectId"], payload["midpointMjdTai"]) else: key = payload["diaObjectId"] - objectids = ["key:key:{}_{}".format(salt, key)] + objectids = [f"key:key:{salt}_{key}"] # if "withupperlim" in payload and str(payload["withupperlim"]) == "True": # withupperlim = True diff --git a/apps/routes/v1/lsst/sso/api.py b/apps/routes/v1/lsst/sso/api.py index 1586278..ce4a889 100644 --- a/apps/routes/v1/lsst/sso/api.py +++ b/apps/routes/v1/lsst/sso/api.py @@ -16,10 +16,7 @@ from flask_restx import Namespace, Resource, fields from apps.routes.v1.lsst.sso.utils import extract_sso_data - -from apps.utils.utils import check_args -from apps.utils.utils import send_tabular_data - +from apps.utils.utils import check_args, send_tabular_data ns = Namespace( "api/v1/sso", diff --git a/apps/routes/v1/lsst/sso/utils.py b/apps/routes/v1/lsst/sso/utils.py index 5a0d22a..32acab6 100644 --- a/apps/routes/v1/lsst/sso/utils.py +++ b/apps/routes/v1/lsst/sso/utils.py @@ -12,19 +12,17 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import pandas as pd import requests from flask import Response -import pandas as pd - -from apps.utils.client import connect_to_hbase_table -from apps.utils.decoding import format_lsst_hbase_output - # from fink_utils.sso.miriade import get_miriade_data # from fink_utils.sso.spins import func_hg1g2_with_spin, estimate_sso_params - from line_profiler import profile +from apps.utils.client import connect_to_hbase_table +from apps.utils.decoding import format_lsst_hbase_output + def resolve_packed(n_or_d): """Resolve all packed names corresponding to input n_or_d""" @@ -32,9 +30,7 @@ def resolve_packed(n_or_d): # Pure quaero implementation r = requests.get( - "https://ssp.imcce.fr/webservices/ssodnet/api/resolver.php?-name=a:EQUAL:{}&-mime=json&-from=FINK".format( - n_or_d - ) + f"https://ssp.imcce.fr/webservices/ssodnet/api/resolver.php?-name=a:EQUAL:{n_or_d}&-mime=json&-from=FINK" ) if r.status_code == 200 and r.json() != []: sso_name = r.json()["data"][0]["name"] @@ -117,18 +113,14 @@ def extract_sso_data(payload: dict) -> pd.DataFrame: if sso_name == "": rep = { "status": "error", - "text": "{} is not a valid name or number according to quaero.\n".format( - id_ - ), + "text": f"{id_} is not a valid name or number according to quaero.\n", } return Response(str(rep), 400) if len(aliases) == 0: rep = { "status": "error", - "text": "We have found 0 packed designation in the aliases for the object {} according to quaero.\n".format( - id_ - ), + "text": f"We have found 0 packed designation in the aliases for the object {id_} according to quaero.\n", } return Response(str(rep), 400) diff --git a/apps/routes/v1/lsst/statistics/api.py b/apps/routes/v1/lsst/statistics/api.py index 707d57a..156b2eb 100644 --- a/apps/routes/v1/lsst/statistics/api.py +++ b/apps/routes/v1/lsst/statistics/api.py @@ -15,10 +15,8 @@ from flask import Response, request from flask_restx import Namespace, Resource, fields -from apps.utils.utils import check_args -from apps.utils.utils import send_tabular_data - from apps.routes.v1.lsst.statistics.utils import get_statistics +from apps.utils.utils import check_args, send_tabular_data ns = Namespace( "api/v1/statistics", "Get statistics about Fink and the Rubin alert stream" diff --git a/apps/routes/v1/lsst/statistics/test.py b/apps/routes/v1/lsst/statistics/test.py index 6a1b0fa..11b888b 100644 --- a/apps/routes/v1/lsst/statistics/test.py +++ b/apps/routes/v1/lsst/statistics/test.py @@ -12,12 +12,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import requests -import pandas as pd - import io import sys +import pandas as pd +import requests + APIURL = sys.argv[1] @@ -29,7 +29,7 @@ def statstest(date="2025", columns="*", output_format="json"): "output-format": output_format, } - r = requests.post("{}/api/v1/statistics".format(APIURL), json=payload) + r = requests.post(f"{APIURL}/api/v1/statistics", json=payload) assert r.status_code == 200, r.content @@ -89,7 +89,7 @@ def test_cols() -> None: if __name__ == "__main__": """ Execute the test suite """ - import sys import doctest + import sys sys.exit(doctest.testmod()[0]) diff --git a/apps/routes/v1/lsst/statistics/utils.py b/apps/routes/v1/lsst/statistics/utils.py index 20035bc..a24a28b 100644 --- a/apps/routes/v1/lsst/statistics/utils.py +++ b/apps/routes/v1/lsst/statistics/utils.py @@ -13,12 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. import pandas as pd +from line_profiler import profile from apps.utils.client import connect_to_hbase_table from apps.utils.decoding import hbase_to_dict -from line_profiler import profile - @profile def get_statistics(payload: dict) -> pd.DataFrame: diff --git a/apps/routes/v1/lsst/tags/api.py b/apps/routes/v1/lsst/tags/api.py index 0fce8d4..2b8e0d8 100644 --- a/apps/routes/v1/lsst/tags/api.py +++ b/apps/routes/v1/lsst/tags/api.py @@ -12,13 +12,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from flask import Response, request, jsonify +from flask import Response, jsonify, request from flask_restx import Namespace, Resource, fields -from apps.utils.utils import check_args -from apps.utils.utils import send_tabular_data - from apps.routes.v1.lsst.tags.utils import extract_object_data, extract_tags +from apps.utils.utils import check_args, send_tabular_data ns = Namespace("api/v1/tags", "Get latest Rubin alerts by tags") @@ -70,7 +68,7 @@ def get(self): return self.post() else: tags, descriptions = extract_tags(True) - out = {k: v for k, v in zip(tags, descriptions)} + out = {k: v for k, v in zip(tags, descriptions, strict=True)} return jsonify(out) @ns.expect(ARGS, location="json", as_dict=True) diff --git a/apps/routes/v1/lsst/tags/utils.py b/apps/routes/v1/lsst/tags/utils.py index 8cc3e94..a5e5902 100644 --- a/apps/routes/v1/lsst/tags/utils.py +++ b/apps/routes/v1/lsst/tags/utils.py @@ -12,19 +12,17 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from flask import Response - -import pandas as pd import importlib import pkgutil -from apps.utils.client import connect_to_hbase_table -from apps.utils.decoding import format_lsst_hbase_output - import fink_filters.rubin.livestream as ffrl - -from line_profiler import profile +import pandas as pd from astropy.time import Time +from flask import Response +from line_profiler import profile + +from apps.utils.client import connect_to_hbase_table +from apps.utils.decoding import format_lsst_hbase_output def extract_tags(with_description=False): @@ -81,11 +79,11 @@ def extract_object_data(payload: dict, return_raw: bool = False) -> pd.DataFrame # Check the tag exists if tag not in extract_tags(): - msg = """ - {} is not a valid tag. Here is the list of tags: - {} + msg = f""" + {tag} is not a valid tag. Here is the list of tags: + {extract_tags()} And you can always retrieve available tags at https://api.lsst.fink-portal.org/api/v1/tags - """.format(tag, extract_tags()) + """ return Response(msg, 400) if "n" not in payload: @@ -114,7 +112,7 @@ def extract_object_data(payload: dict, return_raw: bool = False) -> pd.DataFrame else: truncated = True - client = connect_to_hbase_table("rubin.tag_{}".format(tag)) + client = connect_to_hbase_table(f"rubin.tag_{tag}") client.setLimit(nalerts) client.setRangeScan(True) diff --git a/apps/routes/v1/ztf/anomaly/api.py b/apps/routes/v1/ztf/anomaly/api.py index 9cb4a37..1136879 100644 --- a/apps/routes/v1/ztf/anomaly/api.py +++ b/apps/routes/v1/ztf/anomaly/api.py @@ -15,10 +15,8 @@ from flask import Response, request from flask_restx import Namespace, Resource, fields -from apps.utils.utils import check_args -from apps.utils.utils import send_tabular_data - from apps.routes.v1.ztf.anomaly.utils import get_anomalous_alerts +from apps.utils.utils import check_args, send_tabular_data ns = Namespace("api/v1/anomaly", "Get alerts tagged as anomaly") diff --git a/apps/routes/v1/ztf/anomaly/test.py b/apps/routes/v1/ztf/anomaly/test.py index 5ad3df3..3b1a562 100644 --- a/apps/routes/v1/ztf/anomaly/test.py +++ b/apps/routes/v1/ztf/anomaly/test.py @@ -12,15 +12,14 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import requests -import pandas as pd -import numpy as np - -from astropy.io import votable - import io -import sys import json +import sys + +import numpy as np +import pandas as pd +import requests +from astropy.io import votable APIURL = sys.argv[1] @@ -37,7 +36,7 @@ def anomalysearch( if cols is not None: payload.update({"columns": cols}) - r = requests.post("{}/api/v1/anomaly".format(APIURL), json=payload) + r = requests.post(f"{APIURL}/api/v1/anomaly", json=payload) assert r.status_code == 200, r.content @@ -110,7 +109,7 @@ def test_query_url() -> None: """ pdf1 = anomalysearch() - url = "{}/api/v1/anomaly?n=10&output-format=json".format(APIURL) + url = f"{APIURL}/api/v1/anomaly?n=10&output-format=json" r = requests.get(url) pdf2 = pd.read_json(io.BytesIO(r.content)) @@ -162,7 +161,7 @@ def test_feature_array() -> None: if __name__ == "__main__": """ Execute the test suite """ - import sys import doctest + import sys sys.exit(doctest.testmod()[0]) diff --git a/apps/routes/v1/ztf/anomaly/utils.py b/apps/routes/v1/ztf/anomaly/utils.py index b575dce..e0604d2 100644 --- a/apps/routes/v1/ztf/anomaly/utils.py +++ b/apps/routes/v1/ztf/anomaly/utils.py @@ -13,14 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. import pandas as pd - from astropy.time import Time +from line_profiler import profile from apps.utils.client import connect_to_hbase_table from apps.utils.decoding import format_hbase_output -from line_profiler import profile - @profile def get_anomalous_alerts(payload: dict) -> pd.DataFrame: diff --git a/apps/routes/v1/ztf/classes/api.py b/apps/routes/v1/ztf/classes/api.py index ae80ad8..a482df7 100644 --- a/apps/routes/v1/ztf/classes/api.py +++ b/apps/routes/v1/ztf/classes/api.py @@ -13,12 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. import pandas as pd - +from fink_utils.xmatch.simbad import get_simbad_labels from flask import Response, json from flask_restx import Namespace, Resource -from fink_utils.xmatch.simbad import get_simbad_labels - ns = Namespace("api/v1/classes", "Get Fink derived class names, and their origin") diff --git a/apps/routes/v1/ztf/conesearch/api.py b/apps/routes/v1/ztf/conesearch/api.py index 8e05016..6925e78 100644 --- a/apps/routes/v1/ztf/conesearch/api.py +++ b/apps/routes/v1/ztf/conesearch/api.py @@ -15,10 +15,8 @@ from flask import Response, request from flask_restx import Namespace, Resource, fields -from apps.utils.utils import check_args -from apps.utils.utils import send_tabular_data - from apps.routes.v1.ztf.conesearch.utils import run_conesearch +from apps.utils.utils import check_args, send_tabular_data ns = Namespace("api/v1/conesearch", "Get object data based on coordinates") diff --git a/apps/routes/v1/ztf/conesearch/profiling.py b/apps/routes/v1/ztf/conesearch/profiling.py index 675d971..0eec9ea 100644 --- a/apps/routes/v1/ztf/conesearch/profiling.py +++ b/apps/routes/v1/ztf/conesearch/profiling.py @@ -22,4 +22,4 @@ payload_extragal = {"ra": 193.821739, "dec": 2.897311, "radius": 10.0} pdf = run_conesearch(payload_extragal) -print("{} objects".format(len(pdf))) +print(f"{len(pdf)} objects") diff --git a/apps/routes/v1/ztf/conesearch/test.py b/apps/routes/v1/ztf/conesearch/test.py index 8a97024..b74c88b 100644 --- a/apps/routes/v1/ztf/conesearch/test.py +++ b/apps/routes/v1/ztf/conesearch/test.py @@ -12,16 +12,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import requests -import pandas as pd -import numpy as np +import io +import sys +import numpy as np +import pandas as pd +import requests from astropy.coordinates import SkyCoord from astropy.io import votable -import io -import sys - APIURL = sys.argv[1] @@ -42,7 +41,7 @@ def conesearch( if columns is not None: payload.update({"columns": columns}) - r = requests.post("{}/api/v1/conesearch".format(APIURL), json=payload) + r = requests.post(f"{APIURL}/api/v1/conesearch", json=payload) assert r.status_code == 200, r.content @@ -133,7 +132,7 @@ def test_conesearch_with_cols() -> None: assert not pdf.empty # i:objectId, plus mandatory i:ra,i:dec, plus computed v:separation - assert len(pdf.columns) == 4, "I count {} columns".format(len(pdf.columns)) + assert len(pdf.columns) == 4, f"I count {len(pdf.columns)} columns" def test_bad_radius_conesearch() -> None: @@ -149,7 +148,7 @@ def test_bad_radius_conesearch() -> None: "output_format": "json", } - r = requests.post("{}/api/v1/conesearch".format(APIURL), json=payload) + r = requests.post(f"{APIURL}/api/v1/conesearch", json=payload) msg = { "status": "error", @@ -194,7 +193,7 @@ def test_bad_request() -> None: """ payload = {"ra": "kfdlkj", "dec": "lkfdjf", "radius": 5, "output_format": "json"} - r = requests.post("{}/api/v1/conesearch".format(APIURL), json=payload) + r = requests.post(f"{APIURL}/api/v1/conesearch", json=payload) msg = { "status": "error", @@ -226,7 +225,7 @@ def test_various_outputs() -> None: if __name__ == "__main__": """ Execute the test suite """ - import sys import doctest + import sys sys.exit(doctest.testmod()[0]) diff --git a/apps/routes/v1/ztf/conesearch/utils.py b/apps/routes/v1/ztf/conesearch/utils.py index 95b70e0..a16f4c4 100644 --- a/apps/routes/v1/ztf/conesearch/utils.py +++ b/apps/routes/v1/ztf/conesearch/utils.py @@ -12,22 +12,19 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from flask import Response - -import pandas as pd -from numpy import pi as nppi -from healpy import query_disc, ang2vec - import astropy.units as u -from astropy.time import Time +import pandas as pd from astropy.coordinates import SkyCoord +from astropy.time import Time +from flask import Response +from healpy import ang2vec, query_disc +from line_profiler import profile +from numpy import pi as nppi from apps.utils.client import connect_to_hbase_table from apps.utils.decoding import format_hbase_output from apps.utils.utils import isoify_time -from line_profiler import profile - @profile def run_conesearch(payload: dict) -> pd.DataFrame: diff --git a/apps/routes/v1/ztf/cutouts/api.py b/apps/routes/v1/ztf/cutouts/api.py index d1fe732..e1919f5 100644 --- a/apps/routes/v1/ztf/cutouts/api.py +++ b/apps/routes/v1/ztf/cutouts/api.py @@ -15,9 +15,8 @@ from flask import Response, request from flask_restx import Namespace, Resource, fields -from apps.utils.utils import check_args - from apps.routes.v1.ztf.cutouts.utils import format_and_send_cutout +from apps.utils.utils import check_args ns = Namespace("api/v1/cutouts", "Get cutout data based on ZTF ID") diff --git a/apps/routes/v1/ztf/cutouts/profiling.py b/apps/routes/v1/ztf/cutouts/profiling.py index d97cd5f..d2b4e36 100644 --- a/apps/routes/v1/ztf/cutouts/profiling.py +++ b/apps/routes/v1/ztf/cutouts/profiling.py @@ -14,9 +14,10 @@ # limitations under the License. """Call format_and_send_cutout""" -from apps.routes.v1.ztf.cutouts.utils import format_and_send_cutout from flask import Flask +from apps.routes.v1.ztf.cutouts.utils import format_and_send_cutout + app = Flask("Profile cutouts") payload = { diff --git a/apps/routes/v1/ztf/cutouts/test.py b/apps/routes/v1/ztf/cutouts/test.py index 8cf4aae..56655eb 100644 --- a/apps/routes/v1/ztf/cutouts/test.py +++ b/apps/routes/v1/ztf/cutouts/test.py @@ -12,16 +12,14 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import requests -import numpy as np +import io +import sys +import numpy as np +import requests from astropy.io import fits - from PIL import Image -import io -import sys - APIURL = sys.argv[1] @@ -54,7 +52,7 @@ def cutouttest( if convolution_kernel is not None: payload.update({"convolution_kernel": convolution_kernel}) - r = requests.post("{}/api/v1/cutouts".format(APIURL), json=payload) + r = requests.post(f"{APIURL}/api/v1/cutouts", json=payload) assert r.status_code == 200, r.content @@ -64,7 +62,7 @@ def cutouttest( elif output_format == "FITS": data = fits.open(io.BytesIO(r.content), ignore_missing_simple=True) elif output_format == "array": - data = r.json()["b:cutout{}_stampData".format(kind)] + data = r.json()[f"b:cutout{kind}_stampData"] return data @@ -190,7 +188,7 @@ def test_candid_cutout() -> None: if __name__ == "__main__": """ Execute the test suite """ - import sys import doctest + import sys sys.exit(doctest.testmod()[0]) diff --git a/apps/routes/v1/ztf/cutouts/utils.py b/apps/routes/v1/ztf/cutouts/utils.py index b63c1b8..5026a68 100644 --- a/apps/routes/v1/ztf/cutouts/utils.py +++ b/apps/routes/v1/ztf/cutouts/utils.py @@ -12,23 +12,21 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from flask import send_file, jsonify, Response - import io import json -import requests import numpy as np +import requests +from flask import Response, jsonify, send_file +from line_profiler import profile from matplotlib import cm from PIL import Image from apps.utils.client import connect_to_hbase_table -from apps.utils.plotting import sigmoid_normalizer, legacy_normalizer, convolve from apps.utils.decoding import format_hbase_output +from apps.utils.plotting import convolve, legacy_normalizer, sigmoid_normalizer from apps.utils.utils import extract_configuration -from line_profiler import profile - @profile def format_and_send_cutout(payload: dict): @@ -174,7 +172,7 @@ def format_and_send_cutout(payload: dict): if "colormap" in payload: colormap = getattr(cm, payload["colormap"]) else: - colormap = lambda x: x # noqa: E731 + colormap = lambda x: x array = np.uint8(colormap(array) * 255) # Convert to PNG @@ -207,14 +205,10 @@ def request_cutout(json_payload, output_format, cutout_api_url): """ if output_format == "FITS": json_payload.update({"return_type": "FITS"}) - r0 = requests.post( - "{}/api/v1/cutouts".format(cutout_api_url), json=json_payload - ) + r0 = requests.post(f"{cutout_api_url}/api/v1/cutouts", json=json_payload) cutout = io.BytesIO(r0.content) elif output_format in ["PNG", "array"]: json_payload.update({"return_type": "array"}) - r0 = requests.post( - "{}/api/v1/cutouts".format(cutout_api_url), json=json_payload - ) + r0 = requests.post(f"{cutout_api_url}/api/v1/cutouts", json=json_payload) cutout = json.loads(r0.content) return cutout diff --git a/apps/routes/v1/ztf/latests/api.py b/apps/routes/v1/ztf/latests/api.py index 9bf36cd..b53e31a 100644 --- a/apps/routes/v1/ztf/latests/api.py +++ b/apps/routes/v1/ztf/latests/api.py @@ -15,10 +15,8 @@ from flask import Response, request from flask_restx import Namespace, Resource, fields -from apps.utils.utils import check_args -from apps.utils.utils import send_tabular_data - from apps.routes.v1.ztf.latests.utils import extract_object_from_class +from apps.utils.utils import check_args, send_tabular_data ns = Namespace("api/v1/latests", "Get object data based their class") diff --git a/apps/routes/v1/ztf/latests/test.py b/apps/routes/v1/ztf/latests/test.py index d69af51..9a386f4 100644 --- a/apps/routes/v1/ztf/latests/test.py +++ b/apps/routes/v1/ztf/latests/test.py @@ -12,15 +12,14 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import requests -import pandas as pd -import numpy as np - -from astropy.io import votable - import io import sys +import numpy as np +import pandas as pd +import requests +from astropy.io import votable + APIURL = sys.argv[1] @@ -45,7 +44,7 @@ def classsearch( if trend is not None: payload.update({"trend": trend}) - r = requests.post("{}/api/v1/latests".format(APIURL), json=payload) + r = requests.post(f"{APIURL}/api/v1/latests", json=payload) assert r.status_code == 200, r.content @@ -100,7 +99,7 @@ def test_simbad_classsearch() -> None: def test_tns_classsearch() -> None: """ Examples - --------- + -------- >>> test_tns_classsearch() """ pdf = classsearch(myclass="(TNS) SN Ia") @@ -224,11 +223,7 @@ def test_query_url() -> None: """ pdf1 = classsearch() - url = ( - "{}/api/v1/latests?class=Early SN Ia candidate&n=10&output-format=json".format( - APIURL - ) - ) + url = f"{APIURL}/api/v1/latests?class=Early SN Ia candidate&n=10&output-format=json" r = requests.get(url) pdf2 = pd.read_json(io.BytesIO(r.content)) @@ -262,7 +257,7 @@ def test_various_outputs() -> None: if __name__ == "__main__": """ Execute the test suite """ - import sys import doctest + import sys sys.exit(doctest.testmod()[0]) diff --git a/apps/routes/v1/ztf/latests/utils.py b/apps/routes/v1/ztf/latests/utils.py index 4a52aa2..0a16b86 100644 --- a/apps/routes/v1/ztf/latests/utils.py +++ b/apps/routes/v1/ztf/latests/utils.py @@ -12,16 +12,14 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from flask import Response - import pandas as pd from astropy.time import Time +from flask import Response +from line_profiler import profile from apps.utils.client import connect_to_hbase_table from apps.utils.decoding import format_hbase_output -from line_profiler import profile - @profile def extract_object_from_class(payload: dict, return_raw: bool = False) -> pd.DataFrame: diff --git a/apps/routes/v1/ztf/metadata/api.py b/apps/routes/v1/ztf/metadata/api.py index 9754406..5d8ad18 100644 --- a/apps/routes/v1/ztf/metadata/api.py +++ b/apps/routes/v1/ztf/metadata/api.py @@ -15,14 +15,12 @@ from flask import Response, request from flask_restx import Namespace, Resource, fields -from apps.utils.utils import check_args -from apps.utils.utils import send_tabular_data - from apps.routes.v1.ztf.metadata.utils import ( post_metadata, retrieve_metadata, retrieve_oid, ) +from apps.utils.utils import check_args, send_tabular_data ns = Namespace("api/v1/metadata", "Get user-defined metadata attached to objects") diff --git a/apps/routes/v1/ztf/metadata/test.py b/apps/routes/v1/ztf/metadata/test.py index 6075a3e..72db7d4 100644 --- a/apps/routes/v1/ztf/metadata/test.py +++ b/apps/routes/v1/ztf/metadata/test.py @@ -12,12 +12,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import urllib.parse -import requests -import pandas as pd - import io import sys +import urllib.parse + +import pandas as pd +import requests APIURL = sys.argv[1] @@ -28,18 +28,14 @@ def get_metadata(objectId=None, internal_name=None, internal_name_encoded=None): """Query metadata using GET""" if objectId is not None: - r = requests.get("{}/api/v1/metadata?objectId={}".format(APIURL, objectId)) + r = requests.get(f"{APIURL}/api/v1/metadata?objectId={objectId}") if internal_name is not None: - r = requests.get( - "{}/api/v1/metadata?internal_name={}".format(APIURL, internal_name) - ) + r = requests.get(f"{APIURL}/api/v1/metadata?internal_name={internal_name}") if internal_name_encoded is not None: r = requests.get( - "{}/api/v1/metadata?internal_name_encoded={}".format( - APIURL, internal_name_encoded - ) + f"{APIURL}/api/v1/metadata?internal_name_encoded={internal_name_encoded}" ) assert r.status_code == 200, r.content @@ -49,15 +45,15 @@ def get_metadata(objectId=None, internal_name=None, internal_name_encoded=None): return pdf -def test_objectId() -> None: +def test_objectid() -> None: """ Examples -------- - >>> test_objectId() + >>> test_objectid() """ pdf = get_metadata(objectId="ZTF23aaaatwl") - assert pdf["d:comments"].values[0] == "coucou", pdf + assert pdf["d:comments"].to_numpy()[0] == "coucou", pdf def test_internal_name() -> None: @@ -69,15 +65,19 @@ def test_internal_name() -> None: pdf = get_metadata( internal_name=urllib.parse.quote_plus("Fink J042203.10+362318.7") ) - oid = pdf["i:objectId"].values[0] + oid = pdf["i:objectId"].to_numpy()[0] assert oid == "ZTF20aahjjjm", oid # Get metadata pdf = get_metadata(objectId=oid) - assert pdf["d:username"].values[0] == "pruzhinskaya", pdf - assert pdf["d:comments"].values[0] == "Candidate to red dwarf flare, credit", pdf - assert pdf["d:internal_name_encoded"].values[0] == "FinkJ042203.10+362318.7", pdf + assert pdf["d:username"].to_numpy()[0] == "pruzhinskaya", pdf + assert pdf["d:comments"].to_numpy()[0] == "Candidate to red dwarf flare, credit", ( + pdf + ) + assert pdf["d:internal_name_encoded"].to_numpy()[0] == "FinkJ042203.10+362318.7", ( + pdf + ) def test_internal_name_encoded() -> None: @@ -90,15 +90,15 @@ def test_internal_name_encoded() -> None: internal_name_encoded=urllib.parse.quote_plus("FinkJ061603.51+080222.8") ) - oid = pdf["i:objectId"].values[0] + oid = pdf["i:objectId"].to_numpy()[0] assert oid == "ZTF17aaagtdb", oid # Get metadata pdf = get_metadata(objectId=oid) - assert pdf["d:username"].values[0] == "pruzhinskaya", pdf - assert pdf["d:comments"].values[0] == "Candidate to CV, credit", pdf - assert pdf["d:internal_name"].values[0] == "Fink J061603.51+080222.8", pdf + assert pdf["d:username"].to_numpy()[0] == "pruzhinskaya", pdf + assert pdf["d:comments"].to_numpy()[0] == "Candidate to CV, credit", pdf + assert pdf["d:internal_name"].to_numpy()[0] == "Fink J061603.51+080222.8", pdf def test_get_all() -> None: @@ -113,7 +113,7 @@ def test_get_all() -> None: if __name__ == "__main__": """ Execute the test suite """ - import sys import doctest + import sys sys.exit(doctest.testmod()[0]) diff --git a/apps/routes/v1/ztf/metadata/utils.py b/apps/routes/v1/ztf/metadata/utils.py index 6de7d9a..fbed7c7 100644 --- a/apps/routes/v1/ztf/metadata/utils.py +++ b/apps/routes/v1/ztf/metadata/utils.py @@ -12,15 +12,14 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from flask import Response import pandas as pd +from flask import Response +from line_profiler import profile from apps.utils.client import connect_to_hbase_table from apps.utils.decoding import hbase_to_dict from apps.utils.utils import extract_configuration -from line_profiler import profile - @profile def post_metadata(payload: dict) -> Response: diff --git a/apps/routes/v1/ztf/objects/api.py b/apps/routes/v1/ztf/objects/api.py index 6184d6c..63c8689 100644 --- a/apps/routes/v1/ztf/objects/api.py +++ b/apps/routes/v1/ztf/objects/api.py @@ -15,10 +15,8 @@ from flask import Response, request from flask_restx import Namespace, Resource, fields -from apps.utils.utils import check_args -from apps.utils.utils import send_tabular_data - from apps.routes.v1.ztf.objects.utils import extract_object_data +from apps.utils.utils import check_args, send_tabular_data ns = Namespace("api/v1/objects", "Get object data based on ZTF ID") diff --git a/apps/routes/v1/ztf/objects/test.py b/apps/routes/v1/ztf/objects/test.py index 8494d2c..2cd69ef 100644 --- a/apps/routes/v1/ztf/objects/test.py +++ b/apps/routes/v1/ztf/objects/test.py @@ -12,13 +12,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import requests -import pandas as pd -import numpy as np - import io import sys +import numpy as np +import pandas as pd +import requests + APIURL = sys.argv[1] # Implement random name generator @@ -45,7 +45,7 @@ def get_an_object( if cutout_kind is not None: payload.update({"cutout-kind": cutout_kind}) - r = requests.post("{}/api/v1/objects".format(APIURL), json=payload) + r = requests.post(f"{APIURL}/api/v1/objects", json=payload) assert r.status_code == 200, r.content @@ -101,7 +101,7 @@ def test_column_selection() -> None: """ pdf = get_an_object(oid=OID, columns="i:jd,i:magpsf") - assert len(pdf.columns) == 2, "I count {} columns".format(len(pdf.columns)) + assert len(pdf.columns) == 2, f"I count {len(pdf.columns)} columns" def test_column_length() -> None: @@ -112,7 +112,7 @@ def test_column_length() -> None: """ pdf = get_an_object(oid=OID) - assert len(pdf.columns) == 132, "I count {} columns".format(len(pdf.columns)) + assert len(pdf.columns) == 132, f"I count {len(pdf.columns)} columns" def test_withupperlim() -> None: @@ -212,15 +212,13 @@ def test_multiple_objects() -> None: n_oids_single += n_oid len_object += len(pdf_) - assert n_oids == n_oids_single, "{} is not equal to {}".format( - n_oids, n_oids_single - ) - assert len_object == len(pdf), "{} is not equal to {}".format(len_object, len(pdf)) + assert n_oids == n_oids_single, f"{n_oids} is not equal to {n_oids_single}" + assert len_object == len(pdf), f"{len_object} is not equal to {len(pdf)}" if __name__ == "__main__": """ Execute the test suite """ - import sys import doctest + import sys sys.exit(doctest.testmod()[0]) diff --git a/apps/routes/v1/ztf/objects/utils.py b/apps/routes/v1/ztf/objects/utils.py index 90a35bc..ada5d2a 100644 --- a/apps/routes/v1/ztf/objects/utils.py +++ b/apps/routes/v1/ztf/objects/utils.py @@ -13,13 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. import pandas as pd +from line_profiler import profile from numpy import array as nparray -from apps.utils.utils import download_cutout from apps.utils.client import connect_to_hbase_table from apps.utils.decoding import format_hbase_output, hbase_to_dict - -from line_profiler import profile +from apps.utils.utils import download_cutout @profile @@ -104,7 +103,7 @@ def extract_object_data(payload: dict) -> pd.DataFrame: axis=1, ) else: - colname = "b:cutout{}_stampData".format(cutout_kind) + colname = f"b:cutout{cutout_kind}_stampData" pdf[colname] = pdf[["i:objectId", "i:candid"]].apply( lambda x: pd.Series( [download_cutout(x.iloc[0], x.iloc[1], cutout_kind)] diff --git a/apps/routes/v1/ztf/resolver/api.py b/apps/routes/v1/ztf/resolver/api.py index 7c68a16..55fd0bb 100644 --- a/apps/routes/v1/ztf/resolver/api.py +++ b/apps/routes/v1/ztf/resolver/api.py @@ -15,10 +15,8 @@ from flask import Response, request from flask_restx import Namespace, Resource, fields -from apps.utils.utils import check_args -from apps.utils.utils import send_tabular_data - from apps.routes.v1.ztf.resolver.utils import resolve_name +from apps.utils.utils import check_args, send_tabular_data ns = Namespace( "api/v1/resolver", "Resolve astronomical names to ZTF alert names, and vice-versa" diff --git a/apps/routes/v1/ztf/resolver/test.py b/apps/routes/v1/ztf/resolver/test.py index 141be63..4862fc5 100644 --- a/apps/routes/v1/ztf/resolver/test.py +++ b/apps/routes/v1/ztf/resolver/test.py @@ -12,14 +12,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import requests -import pandas as pd - -from astropy.io import votable - import io import sys +import pandas as pd +import requests +from astropy.io import votable + APIURL = sys.argv[1] @@ -41,7 +40,7 @@ def resolver(resolver="", name="", nmax=None, reverse=None, output_format="json" } ) - r = requests.post("{}/api/v1/resolver".format(APIURL), json=payload) + r = requests.post(f"{APIURL}/api/v1/resolver", json=payload) assert r.status_code == 200, r.content @@ -248,7 +247,7 @@ def test_reverse_ssodnet_resolver() -> None: if __name__ == "__main__": """ Execute the test suite """ - import sys import doctest + import sys sys.exit(doctest.testmod()[0]) diff --git a/apps/routes/v1/ztf/resolver/utils.py b/apps/routes/v1/ztf/resolver/utils.py index bb52e19..6306228 100644 --- a/apps/routes/v1/ztf/resolver/utils.py +++ b/apps/routes/v1/ztf/resolver/utils.py @@ -13,15 +13,15 @@ # See the License for the specific language governing permissions and # limitations under the License. import io -import requests + import pandas as pd +import requests +from line_profiler import profile from numpy import unique as npunique from apps.utils.client import connect_to_hbase_table from apps.utils.decoding import hbase_to_dict -from line_profiler import profile - @profile def resolve_name(payload: dict) -> pd.DataFrame: diff --git a/apps/routes/v1/ztf/schema/api.py b/apps/routes/v1/ztf/schema/api.py index 2753da3..4325779 100644 --- a/apps/routes/v1/ztf/schema/api.py +++ b/apps/routes/v1/ztf/schema/api.py @@ -12,9 +12,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import requests import pandas as pd - +import requests from flask import Response, json from flask_restx import Namespace, Resource @@ -385,23 +384,28 @@ def get(self): "ZTF original fields (i:)": { i: {"type": j, "doc": k} for i, j, k in zip( - ztf_candidate.name, ztf_candidate.type, ztf_candidate.doc + ztf_candidate.name, + ztf_candidate.type, + ztf_candidate.doc, + strict=True, ) }, "ZTF original cutouts (b:)": { i: {"type": j, "doc": k} - for i, j, k in zip(ztf_cutouts.name, ztf_cutouts.type, ztf_cutouts.doc) + for i, j, k in zip( + ztf_cutouts.name, ztf_cutouts.type, ztf_cutouts.doc, strict=True + ) }, "Fink science module outputs (d:)": { i: {"type": j, "doc": k} for i, j, k in zip( - fink_science.name, fink_science.type, fink_science.doc + fink_science.name, fink_science.type, fink_science.doc, strict=True ) }, "Fink on-the-fly added values (v:)": { i: {"type": j, "doc": k} for i, j, k in zip( - fink_derived.name, fink_derived.type, fink_derived.doc + fink_derived.name, fink_derived.type, fink_derived.doc, strict=True ) }, } diff --git a/apps/routes/v1/ztf/schema/test.py b/apps/routes/v1/ztf/schema/test.py index 7355bb7..18d5be1 100644 --- a/apps/routes/v1/ztf/schema/test.py +++ b/apps/routes/v1/ztf/schema/test.py @@ -12,18 +12,18 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import requests -import pandas as pd - import io import sys +import pandas as pd +import requests + APIURL = sys.argv[1] def get_fields_per_family(): """Get fields exposed in /api/v1/schema""" - r = requests.get("{}/api/v1/schema".format(APIURL)) + r = requests.get(f"{APIURL}/api/v1/schema") assert r.status_code == 200, r.content @@ -187,7 +187,7 @@ def test_recent_object() -> None: """ # Need a class that bring objects frequently (to have new cols) r = requests.post( - "{}/api/v1/latests".format(APIURL), + f"{APIURL}/api/v1/latests", json={"class": "SN candidate", "n": 1, "columns": "i:objectId"}, ) @@ -196,7 +196,7 @@ def test_recent_object() -> None: objectId = r.json()[0]["i:objectId"] # Not that we discard images on purpose for speed r2 = requests.post( - "{}/api/v1/objects".format(APIURL), + f"{APIURL}/api/v1/objects", json={ "objectId": objectId, "columns": "*", @@ -221,7 +221,7 @@ def test_old_object() -> None: """ objectId = "ZTF21abfmbix" r = requests.post( - "{}/api/v1/objects".format(APIURL), + f"{APIURL}/api/v1/objects", json={ "objectId": objectId, "columns": "*", @@ -240,7 +240,7 @@ def test_old_object() -> None: if __name__ == "__main__": """ Execute the test suite """ - import sys import doctest + import sys sys.exit(doctest.testmod()[0]) diff --git a/apps/routes/v1/ztf/skymap/api.py b/apps/routes/v1/ztf/skymap/api.py index 6dcea84..3fbd360 100644 --- a/apps/routes/v1/ztf/skymap/api.py +++ b/apps/routes/v1/ztf/skymap/api.py @@ -15,10 +15,8 @@ from flask import Response, request from flask_restx import Namespace, Resource, fields -from apps.utils.utils import check_args -from apps.utils.utils import send_tabular_data - from apps.routes.v1.ztf.skymap.utils import search_in_skymap +from apps.utils.utils import check_args, send_tabular_data ns = Namespace("api/v1/skymap", "Return Fink/ZTF alerts within a GW skymap") diff --git a/apps/routes/v1/ztf/skymap/profiling.py b/apps/routes/v1/ztf/skymap/profiling.py index 890ae02..7ba46ba 100644 --- a/apps/routes/v1/ztf/skymap/profiling.py +++ b/apps/routes/v1/ztf/skymap/profiling.py @@ -19,4 +19,4 @@ payload = {"event_name": "S230709bi", "credible_level": 0.2} pdf = search_in_skymap(payload) -print("{} objects".format(len(pdf))) +print(f"{len(pdf)} objects") diff --git a/apps/routes/v1/ztf/skymap/test.py b/apps/routes/v1/ztf/skymap/test.py index 5cc8351..b3f9f32 100644 --- a/apps/routes/v1/ztf/skymap/test.py +++ b/apps/routes/v1/ztf/skymap/test.py @@ -12,12 +12,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import requests -import pandas as pd - import io import sys +import pandas as pd +import requests + APIURL = sys.argv[1] @@ -44,7 +44,7 @@ def bayestartest( "output-format": output_format, } - r = requests.post("{}/api/v1/skymap".format(APIURL), json=payload) + r = requests.post(f"{APIURL}/api/v1/skymap", json=payload) assert r.status_code == 200, r.content @@ -87,7 +87,7 @@ def test_name_bayestar() -> None: if __name__ == "__main__": """ Execute the test suite """ - import sys import doctest + import sys sys.exit(doctest.testmod()[0]) diff --git a/apps/routes/v1/ztf/skymap/utils.py b/apps/routes/v1/ztf/skymap/utils.py index 872b9e9..cedf1c8 100644 --- a/apps/routes/v1/ztf/skymap/utils.py +++ b/apps/routes/v1/ztf/skymap/utils.py @@ -12,22 +12,20 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import io import gzip -import requests - -from astropy.io import fits -from astropy.time import Time +import io import healpy as hp -import pandas as pd import numpy as np +import pandas as pd +import requests +from astropy.io import fits +from astropy.time import Time +from line_profiler import profile from apps.utils.client import connect_to_hbase_table from apps.utils.decoding import format_hbase_output -from line_profiler import profile - @profile def search_in_skymap(payload: dict) -> pd.DataFrame: diff --git a/apps/routes/v1/ztf/sso/api.py b/apps/routes/v1/ztf/sso/api.py index 1ad766d..ea33b13 100644 --- a/apps/routes/v1/ztf/sso/api.py +++ b/apps/routes/v1/ztf/sso/api.py @@ -16,10 +16,7 @@ from flask_restx import Namespace, Resource, fields from apps.routes.v1.ztf.sso.utils import extract_sso_data - -from apps.utils.utils import check_args -from apps.utils.utils import send_tabular_data - +from apps.utils.utils import check_args, send_tabular_data ns = Namespace("api/v1/sso", "Get Solar System object data based on their ID") diff --git a/apps/routes/v1/ztf/sso/test.py b/apps/routes/v1/ztf/sso/test.py index a0cc890..d494bab 100644 --- a/apps/routes/v1/ztf/sso/test.py +++ b/apps/routes/v1/ztf/sso/test.py @@ -12,13 +12,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import requests -import pandas as pd -import numpy as np - import io import sys +import numpy as np +import pandas as pd +import requests + APIURL = sys.argv[1] @@ -45,7 +45,7 @@ def ssosearch( if cutout_kind is not None: payload.update({"cutout-kind": cutout_kind}) - r = requests.post("{}/api/v1/sso".format(APIURL), json=payload) + r = requests.post(f"{APIURL}/api/v1/sso", json=payload) assert r.status_code == expected_status, r.content @@ -259,7 +259,7 @@ def test_with_ephem_multiple_ssosearch() -> None: if __name__ == "__main__": """ Execute the test suite """ - import sys import doctest + import sys sys.exit(doctest.testmod()[0]) diff --git a/apps/routes/v1/ztf/sso/utils.py b/apps/routes/v1/ztf/sso/utils.py index d82223b..731586e 100644 --- a/apps/routes/v1/ztf/sso/utils.py +++ b/apps/routes/v1/ztf/sso/utils.py @@ -12,24 +12,21 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import numpy as np +import pandas as pd import requests +from fink_utils.sso.miriade import get_miriade_data +from fink_utils.sso.spins import estimate_sso_params, func_shg1g2 from flask import Response +from line_profiler import profile -import pandas as pd -import numpy as np - +from apps.utils.client import connect_to_hbase_table +from apps.utils.decoding import format_hbase_output from apps.utils.utils import ( download_cutout, - resolve_sso_name_to_ssnamenr, resolve_sso_name, + resolve_sso_name_to_ssnamenr, ) -from apps.utils.client import connect_to_hbase_table -from apps.utils.decoding import format_hbase_output - -from fink_utils.sso.miriade import get_miriade_data -from fink_utils.sso.spins import func_shg1g2, estimate_sso_params - -from line_profiler import profile @profile @@ -72,14 +69,16 @@ def extract_sso_data(payload: dict) -> pd.DataFrame: ): with_cutouts = True - if with_cutouts and truncated: - # Check mandatory fields - if "i:objectId" not in cols or "i:candid" not in cols: - rep = { - "status": "error", - "text": "You need to add 'i:objectId,i:candid' to the columns.\n", - } - return Response(str(rep), 400) + if ( + with_cutouts + and truncated + and (("i:objectId" not in cols) or ("i:candid" not in cols)) + ): + rep = { + "status": "error", + "text": "You need to add 'i:objectId,i:candid' to the columns.\n", + } + return Response(str(rep), 400) if truncated and "i:ssnamenr" not in cols: # For name resolving, i:ssnamenr must be here @@ -112,9 +111,7 @@ def extract_sso_data(payload: dict) -> pd.DataFrame: start = id_[0:6] stop = id_[6:] r = requests.get( - "https://api.ssodnet.imcce.fr/quaero/1/sso?q={} {}&type=Comet".format( - start, stop - ) + f"https://api.ssodnet.imcce.fr/quaero/1/sso?q={start} {stop}&type=Comet" ) if r.status_code == 200 and r.json() != []: sso_name = r.json()["data"][0]["name"] @@ -131,9 +128,7 @@ def extract_sso_data(payload: dict) -> pd.DataFrame: if not isinstance(sso_number, int) and not isinstance(sso_name, str): rep = { "status": "error", - "text": "{} is not a valid name or number according to quaero.\n".format( - n_or_d - ), + "text": f"{n_or_d} is not a valid name or number according to quaero.\n", } return Response(str(rep), 400) @@ -194,7 +189,7 @@ def extract_sso_data(payload: dict) -> pd.DataFrame: return Response(str(rep), 400) # get cutouts - colname = "b:cutout{}_stampData".format(cutout_kind) + colname = f"b:cutout{cutout_kind}_stampData" cutouts = [] for _, row in pdf.iterrows(): cutouts.append( @@ -250,9 +245,9 @@ def extract_sso_data(payload: dict) -> pd.DataFrame: cond = pdf["i:fid"] == filt model = func_shg1g2( [phase[cond], ra[cond], dec[cond]], - outdic["H_{}".format(filt)], - outdic["G1_{}".format(filt)], - outdic["G2_{}".format(filt)], + outdic[f"H_{filt}"], + outdic[f"G1_{filt}"], + outdic[f"G2_{filt}"], outdic["R"], np.deg2rad(outdic["alpha0"]), np.deg2rad(outdic["delta0"]), diff --git a/apps/routes/v1/ztf/ssobulk/api.py b/apps/routes/v1/ztf/ssobulk/api.py index ec97c6b..0985df8 100644 --- a/apps/routes/v1/ztf/ssobulk/api.py +++ b/apps/routes/v1/ztf/ssobulk/api.py @@ -14,13 +14,10 @@ # limitations under the License. from flask import Response, request from flask_restx import Namespace, Resource, fields - from pandas import DataFrame -from apps.utils.utils import check_args -from apps.utils.utils import send_tabular_data - from apps.routes.v1.ztf.ssobulk.utils import get_lc +from apps.utils.utils import check_args, send_tabular_data ns = Namespace("api/v1/ssobulk", "Get all Fink/ZTF SSO lightcurves in once") diff --git a/apps/routes/v1/ztf/ssobulk/utils.py b/apps/routes/v1/ztf/ssobulk/utils.py index 05bf2d2..fe477fd 100644 --- a/apps/routes/v1/ztf/ssobulk/utils.py +++ b/apps/routes/v1/ztf/ssobulk/utils.py @@ -12,14 +12,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from flask import Response - import io -import yaml -import requests import pandas as pd - +import requests +import yaml +from flask import Response from line_profiler import profile @@ -39,7 +37,8 @@ def get_lc(payload: dict) -> pd.DataFrame: out: pandas dataframe """ # Need to profile compared to pyarrow - input_args = yaml.load(open("config.yml"), yaml.Loader) + with open("config.yml") as f: + input_args = yaml.load(f, yaml.Loader) r = requests.get( "{}/sso_ztf_lc_aggregated_with_ssoft_202601_with_residuals_singlefile.parquet?op=OPEN&user.name={}&namenoderpcaddress={}".format( input_args["WEBHDFS"], diff --git a/apps/routes/v1/ztf/ssocand/api.py b/apps/routes/v1/ztf/ssocand/api.py index 2b2e36b..3984320 100644 --- a/apps/routes/v1/ztf/ssocand/api.py +++ b/apps/routes/v1/ztf/ssocand/api.py @@ -15,10 +15,8 @@ from flask import Response, request from flask_restx import Namespace, Resource, fields -from apps.utils.utils import check_args -from apps.utils.utils import send_tabular_data - from apps.routes.v1.ztf.ssocand.utils import get_ssocand +from apps.utils.utils import check_args, send_tabular_data ns = Namespace("api/v1/ssocand", "Get data about Solar System candidates found by Fink") diff --git a/apps/routes/v1/ztf/ssocand/test.py b/apps/routes/v1/ztf/ssocand/test.py index accb29b..2410659 100644 --- a/apps/routes/v1/ztf/ssocand/test.py +++ b/apps/routes/v1/ztf/ssocand/test.py @@ -12,15 +12,14 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import requests -import pandas as pd -import numpy as np - -from astropy.time import Time - import io import sys +import numpy as np +import pandas as pd +import requests +from astropy.time import Time + APIURL = sys.argv[1] @@ -44,7 +43,7 @@ def ssocandsearch( if maxnumber is not None: payload.update({"maxnumber": maxnumber}) - r = requests.post("{}/api/v1/ssocand".format(APIURL), json=payload) + r = requests.post(f"{APIURL}/api/v1/ssocand", json=payload) assert r.status_code == 200, r.content @@ -113,7 +112,7 @@ def test_lightcurves_traj() -> None: assert "d:ssoCandId" in pdf.columns nobjects = len(np.unique(pdf["d:ssoCandId"])) - assert nobjects == 1, "Expecting 1 object, but found {}".format(nobjects) + assert nobjects == 1, f"Expecting 1 object, but found {nobjects}" def test_time_boundaries() -> None: @@ -134,7 +133,7 @@ def test_time_boundaries() -> None: if __name__ == "__main__": """ Execute the test suite """ - import sys import doctest + import sys sys.exit(doctest.testmod()[0]) diff --git a/apps/routes/v1/ztf/ssocand/utils.py b/apps/routes/v1/ztf/ssocand/utils.py index 8f0e9ec..b8ec702 100644 --- a/apps/routes/v1/ztf/ssocand/utils.py +++ b/apps/routes/v1/ztf/ssocand/utils.py @@ -13,13 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. import pandas as pd - from astropy.time import Time +from line_profiler import profile from apps.utils.client import connect_to_hbase_table -from apps.utils.decoding import hbase_to_dict, convert_datatype, hbase_type_converter - -from line_profiler import profile +from apps.utils.decoding import convert_datatype, hbase_to_dict, hbase_type_converter @profile @@ -42,10 +40,7 @@ def get_ssocand(payload: dict) -> pd.DataFrame: else: trajectory_id = None - if "maxnumber" in payload: - maxnumber = payload["maxnumber"] - else: - maxnumber = 10000 + maxnumber = payload.get("maxnumber", 10000) payload_name = payload["kind"] diff --git a/apps/routes/v1/ztf/ssoft/api.py b/apps/routes/v1/ztf/ssoft/api.py index 83e5abe..15d2d73 100644 --- a/apps/routes/v1/ztf/ssoft/api.py +++ b/apps/routes/v1/ztf/ssoft/api.py @@ -14,13 +14,10 @@ # limitations under the License. from flask import Response, request from flask_restx import Namespace, Resource, fields - from pandas import DataFrame -from apps.utils.utils import check_args -from apps.utils.utils import send_tabular_data - from apps.routes.v1.ztf.ssoft.utils import get_ssoft +from apps.utils.utils import check_args, send_tabular_data ns = Namespace("api/v1/ssoft", "Get the Solar System Object Fink Table (SSoFT)") diff --git a/apps/routes/v1/ztf/ssoft/profiling.py b/apps/routes/v1/ztf/ssoft/profiling.py index afe134f..6c6db47 100644 --- a/apps/routes/v1/ztf/ssoft/profiling.py +++ b/apps/routes/v1/ztf/ssoft/profiling.py @@ -15,6 +15,7 @@ """Call get_ssoft""" import pandas as pd + from apps.routes.v1.ztf.ssoft.utils import get_ssoft payload = {"version": "2024.11", "flavor": "SHG1G2"} diff --git a/apps/routes/v1/ztf/ssoft/test.py b/apps/routes/v1/ztf/ssoft/test.py index 659544b..32a1029 100644 --- a/apps/routes/v1/ztf/ssoft/test.py +++ b/apps/routes/v1/ztf/ssoft/test.py @@ -12,14 +12,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import requests import datetime - -import pandas as pd - import io import sys +import pandas as pd +import requests + APIURL = sys.argv[1] @@ -69,7 +68,7 @@ def ssoftsearch( } ) - r = requests.post("{}/api/v1/ssoft".format(APIURL), json=payload) + r = requests.post(f"{APIURL}/api/v1/ssoft", json=payload) assert r.status_code == 200, r.content @@ -94,8 +93,8 @@ def default_ssoft() -> None: assert not pdf.empty - now = datetime.datetime.now() - current_date = "{}.{:02d}".format(now.year, now.month) + now = datetime.datetime.now(tz=datetime.timezone.utc) + current_date = f"{now.year}.{now.month:02d}" assert pdf["version"].to_numpy()[0] == current_date @@ -145,9 +144,7 @@ def test_schema() -> None: assert not_in_pdf == [], not_in_pdf assert not_in_schema == [], not_in_schema - msg = "Found {} entries in the DataFrame and {} entries in the schema.".format( - len(pdf.columns), len(schema) - ) + msg = f"Found {len(pdf.columns)} entries in the DataFrame and {len(schema)} entries in the schema." assert set(schema.keys()) == set(pdf.columns), msg @@ -160,7 +157,7 @@ def compare_schema() -> None: schema1 = ssoftsearch(schema=True, flavor="SSHG1G2", output_format="json") # get the schema - r = requests.get("{}/api/v1/ssoft?schema=True&flavor=SSHG1G2".format(APIURL)) + r = requests.get(f"{APIURL}/api/v1/ssoft?schema=True&flavor=SSHG1G2") schema2 = r.json() keys1 = set(schema1.keys()) @@ -182,7 +179,7 @@ def check_sshg1g2() -> None: if __name__ == "__main__": """ Execute the test suite """ - import sys import doctest + import sys sys.exit(doctest.testmod()[0]) diff --git a/apps/routes/v1/ztf/ssoft/utils.py b/apps/routes/v1/ztf/ssoft/utils.py index 6dc4d9d..8360cd6 100644 --- a/apps/routes/v1/ztf/ssoft/utils.py +++ b/apps/routes/v1/ztf/ssoft/utils.py @@ -12,16 +12,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from flask import Response - +import datetime import io import json -import yaml -import requests -import datetime import pandas as pd - +import requests +import yaml from fink_utils.sso.ssoft import ( COLUMNS, COLUMNS_HG, @@ -29,7 +26,7 @@ COLUMNS_SHG1G2, COLUMNS_SOCCA, ) - +from flask import Response from line_profiler import profile @@ -94,7 +91,7 @@ def get_ssoft(payload: dict) -> pd.DataFrame: } return Response(str(rep), 400) else: - now = datetime.datetime.now() + now = datetime.datetime.now(tz=datetime.timezone.utc) version = f"{now.year}.{now.month:02d}" if "flavor" in payload: @@ -109,7 +106,8 @@ def get_ssoft(payload: dict) -> pd.DataFrame: flavor = "SHG1G2" # Need to profile compared to pyarrow - input_args = yaml.load(open("config.yml"), yaml.Loader) + with open("config.yml") as f: + input_args = yaml.load(f, yaml.Loader) r = requests.get( "{}/SSOFT/ssoft_{}_{}.parquet?op=OPEN&user.name={}&namenoderpcaddress={}".format( input_args["WEBHDFS"], diff --git a/apps/routes/v1/ztf/statistics/api.py b/apps/routes/v1/ztf/statistics/api.py index c37be90..d29e94e 100644 --- a/apps/routes/v1/ztf/statistics/api.py +++ b/apps/routes/v1/ztf/statistics/api.py @@ -15,10 +15,8 @@ from flask import Response, request from flask_restx import Namespace, Resource, fields -from apps.utils.utils import check_args -from apps.utils.utils import send_tabular_data - from apps.routes.v1.ztf.statistics.utils import get_statistics +from apps.utils.utils import check_args, send_tabular_data ns = Namespace( "api/v1/statistics", "Get statistics about Fink and the ZTF alert stream" diff --git a/apps/routes/v1/ztf/statistics/test.py b/apps/routes/v1/ztf/statistics/test.py index 80fdefb..1919422 100644 --- a/apps/routes/v1/ztf/statistics/test.py +++ b/apps/routes/v1/ztf/statistics/test.py @@ -12,12 +12,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import requests -import pandas as pd - import io import sys +import pandas as pd +import requests + APIURL = sys.argv[1] @@ -29,7 +29,7 @@ def statstest(date="2021", columns="*", output_format="json"): "output-format": output_format, } - r = requests.post("{}/api/v1/statistics".format(APIURL), json=payload) + r = requests.post(f"{APIURL}/api/v1/statistics", json=payload) assert r.status_code == 200, r.content @@ -89,7 +89,7 @@ def test_cols() -> None: if __name__ == "__main__": """ Execute the test suite """ - import sys import doctest + import sys sys.exit(doctest.testmod()[0]) diff --git a/apps/routes/v1/ztf/statistics/utils.py b/apps/routes/v1/ztf/statistics/utils.py index 26fed09..1add345 100644 --- a/apps/routes/v1/ztf/statistics/utils.py +++ b/apps/routes/v1/ztf/statistics/utils.py @@ -13,12 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. import pandas as pd +from line_profiler import profile from apps.utils.client import connect_to_hbase_table from apps.utils.decoding import hbase_to_dict -from line_profiler import profile - @profile def get_statistics(payload: dict) -> pd.DataFrame: @@ -35,10 +34,7 @@ def get_statistics(payload: dict) -> pd.DataFrame: ---------- out: pandas dataframe """ - if "columns" in payload: - cols = payload["columns"] - else: - cols = "*" + cols = payload.get("columns", "*") client = connect_to_hbase_table("statistics_class") if "schema" in payload and str(payload["schema"]) == "True": diff --git a/apps/routes/v1/ztf/template/api.py b/apps/routes/v1/ztf/template/api.py index 837d787..1395e6e 100644 --- a/apps/routes/v1/ztf/template/api.py +++ b/apps/routes/v1/ztf/template/api.py @@ -15,10 +15,8 @@ from flask import Response, request from flask_restx import Namespace, Resource, fields -from apps.utils.utils import check_args -from apps.utils.utils import send_tabular_data - from apps.routes.v1.ztf.template.utils import my_function +from apps.utils.utils import check_args, send_tabular_data ns = Namespace("api/v1/template", "Template") diff --git a/apps/routes/v1/ztf/template/utils.py b/apps/routes/v1/ztf/template/utils.py index 30eae36..7991fae 100644 --- a/apps/routes/v1/ztf/template/utils.py +++ b/apps/routes/v1/ztf/template/utils.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. import pandas as pd - from line_profiler import profile diff --git a/apps/routes/v1/ztf/tracklet/api.py b/apps/routes/v1/ztf/tracklet/api.py index 70d024b..7766cab 100644 --- a/apps/routes/v1/ztf/tracklet/api.py +++ b/apps/routes/v1/ztf/tracklet/api.py @@ -15,10 +15,8 @@ from flask import Response, request from flask_restx import Namespace, Resource, fields -from apps.utils.utils import check_args -from apps.utils.utils import send_tabular_data - from apps.routes.v1.ztf.tracklet.utils import get_tracklet +from apps.utils.utils import check_args, send_tabular_data ns = Namespace("api/v1/tracklet", "Get satellites and debris candidates") diff --git a/apps/routes/v1/ztf/tracklet/test.py b/apps/routes/v1/ztf/tracklet/test.py index 3da3289..e8d7a2f 100644 --- a/apps/routes/v1/ztf/tracklet/test.py +++ b/apps/routes/v1/ztf/tracklet/test.py @@ -12,15 +12,14 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import requests -import pandas as pd -import numpy as np - -from astropy.time import Time - import io import sys +import numpy as np +import pandas as pd +import requests +from astropy.time import Time + APIURL = sys.argv[1] @@ -28,7 +27,7 @@ def trackletsearch(date="2021-08-10", columns="*", output_format="json"): """Perform a tracklet search in the Science Portal using the Fink REST API""" payload = {"date": date, "columns": columns, "output-format": output_format} - r = requests.post("{}/api/v1/tracklet".format(APIURL), json=payload) + r = requests.post(f"{APIURL}/api/v1/tracklet", json=payload) assert r.status_code == 200, r.content @@ -85,7 +84,7 @@ def test_single_tracklet() -> None: if __name__ == "__main__": """ Execute the test suite """ - import sys import doctest + import sys sys.exit(doctest.testmod()[0]) diff --git a/apps/routes/v1/ztf/tracklet/utils.py b/apps/routes/v1/ztf/tracklet/utils.py index 5699416..4eb847f 100644 --- a/apps/routes/v1/ztf/tracklet/utils.py +++ b/apps/routes/v1/ztf/tracklet/utils.py @@ -13,12 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. from flask import Response +from line_profiler import profile from apps.utils.client import connect_to_hbase_table from apps.utils.decoding import format_hbase_output -from line_profiler import profile - @profile def get_tracklet(payload: dict): diff --git a/apps/utils/client.py b/apps/utils/client.py index 3699119..036f1d0 100644 --- a/apps/utils/client.py +++ b/apps/utils/client.py @@ -14,14 +14,12 @@ # limitations under the License. """Utilities to work with the Fink HBase client""" -from py4j.java_gateway import JavaGateway - import numpy as np +from line_profiler import profile +from py4j.java_gateway import JavaGateway from apps.utils.utils import extract_configuration -from line_profiler import profile - @profile def connect_to_hbase_table( diff --git a/apps/utils/decoding.py b/apps/utils/decoding.py index 30737f8..d7062db 100644 --- a/apps/utils/decoding.py +++ b/apps/utils/decoding.py @@ -14,18 +14,18 @@ # limitations under the License. """Utilities to decode data from the HBase client""" -from py4j.java_gateway import JavaGateway import json -import pandas as pd -import numpy as np import logging -from astropy.time import Time +import numpy as np +import pandas as pd from astropy.coordinates import SkyCoord, get_constellation - +from astropy.time import Time from fink_filters.ztf.classification import extract_fink_classification_ - from line_profiler import profile +from py4j.java_gateway import JavaGateway + +_LOG = logging.getLogger(__name__) pd.set_option("future.no_silent_downcasting", True) @@ -104,7 +104,7 @@ def format_hbase_output( hbase_type_converter[schema_client.type(col)], ) except KeyError: - logging.warning("Cannot cast columns {} -- not found in schema".format(col)) + _LOG.warning(f"Cannot cast columns {col} -- not found in schema") # Booleans pdfs = pdfs.replace(to_replace={"true": True, "false": False}) diff --git a/apps/utils/plotting.py b/apps/utils/plotting.py index 11b23b0..4639873 100644 --- a/apps/utils/plotting.py +++ b/apps/utils/plotting.py @@ -13,11 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. import numpy as np - from astropy.convolution import Box2DKernel, Gaussian2DKernel from astropy.convolution import convolve as astropy_convolve from astropy.visualization import AsymmetricPercentileInterval, simple_norm - from line_profiler import profile diff --git a/apps/utils/utils.py b/apps/utils/utils.py index 49ba26a..5c164b0 100644 --- a/apps/utils/utils.py +++ b/apps/utils/utils.py @@ -16,22 +16,20 @@ import io import json -import yaml -import requests import logging -from flask import Response - -from astropy.table import Table -from astropy.io import votable -from astropy.time import Time - import numpy as np - +import requests import rocks - +import yaml +from astropy.io import votable +from astropy.table import Table +from astropy.time import Time +from flask import Response from line_profiler import profile +_LOG = logging.getLogger(__name__) + def extract_configuration(filename): """Extract user defined configuration @@ -46,7 +44,8 @@ def extract_configuration(filename): out: dict Dictionary with user defined values. """ - config = yaml.load(open("config.yml"), yaml.Loader) + with open("config.yml") as f: + config = yaml.load(f, yaml.Loader) if config["HOST"].endswith(".org"): config["APIURL"] = "https://" + config["HOST"] else: @@ -56,7 +55,7 @@ def extract_configuration(filename): @profile def download_cutout(objectId, candid, kind): - """ """ + """Wrapper around /api/v1/cutouts""" config = extract_configuration("config.yml") r = requests.post( @@ -72,26 +71,20 @@ def download_cutout(objectId, candid, kind): data = json.loads(r.content) else: # TODO: different return based on `kind`? - logging.warning( - "Cutout retrieval failed with status {}: {}".format(r.status_code, r.text) - ) + _LOG.warning(f"Cutout retrieval failed with status {r.status_code}: {r.text}") return [] if kind != "All": - return data["b:cutout{}_stampData".format(kind)] + return data[f"b:cutout{kind}_stampData"] else: return [ - data["b:cutout{}_stampData".format(k)] + data[f"b:cutout{k}_stampData"] for k in ["Science", "Template", "Difference"] ] def check_args(args: list, payload: dict) -> dict: - """Check all required arguments have been supplied - - Parameters - ---------- - """ + """Check all required arguments have been supplied""" required_args = [k for k in args if args[k].required is True] for required_arg in required_args: if required_arg not in payload: diff --git a/config_prometheus.py b/config_prometheus.py index 19ecfd7..5eecceb 100644 --- a/config_prometheus.py +++ b/config_prometheus.py @@ -12,9 +12,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import glob import os import uuid -import glob + from prometheus_flask_exporter.multiprocess import GunicornPrometheusMetrics @@ -24,7 +25,7 @@ def when_ready(server): if path: for f in glob.glob(os.path.join(path, "*.db")): os.remove(f) - PORT = int(os.getenv("PROMETHEUS_METRIC_PORT", 9090)) + PORT = int(os.getenv("PROMETHEUS_METRIC_PORT", "9090")) GunicornPrometheusMetrics.start_http_server_when_ready(PORT) @@ -69,7 +70,7 @@ def add_id(self, worker_id): def child_exit(server, worker): """ - called when a worker exits. + Called when a worker exits. Returns the worker_id to the pool so it can be reused, and marks the worker process ad dead for prometheus monitoring . @@ -84,18 +85,18 @@ def child_exit(server, worker): def pre_fork(server, worker): """ - called by gunicorn master before a worker is forked. Assigns + Called by gunicorn master before a worker is forked. Assigns a unique worker_id from the pool to the worker . :param server: Gunicorn master server instance :param worker: Gunicorn worker instance about to be forked """ - setattr(worker, "worker_id", gunicorn_worker_ids_pool.get_id()) + worker.worker_id = gunicorn_worker_ids_pool.get_id() def post_fork(server, worker): """ - called by gunicorn master after a worker is forked. Expose + Called by gunicorn master after a worker is forked. Expose the worker_id in the env for the worker process . :param server: Gunicorn master server instance diff --git a/lint.sh b/lint.sh new file mode 100755 index 0000000..5d1bfb8 --- /dev/null +++ b/lint.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +pip install ruff --upgrade + +ruff check --fix . +ruff format .