From 65fe24e1df6efb36a1376f9f1a31ddaafc5ef2b9 Mon Sep 17 00:00:00 2001 From: theoniko Date: Sat, 19 Sep 2026 10:37:57 +0300 Subject: [PATCH] Replace datetime.utcnow() with datetime.now(timezone.utc) --- bugbug/bugzilla.py | 4 +- bugbug/models/accessibility.py | 4 +- bugbug/models/backout.py | 10 +++- bugbug/models/regressor.py | 14 +++-- bugbug/phabricator.py | 6 +- bugbug/repository.py | 6 +- functions/sync-review-comments-db/models.py | 22 +++++-- scripts/bug_retriever.py | 6 +- scripts/commit_classifier.py | 4 +- scripts/generate_landings_risk_report.py | 60 ++++++++++++-------- scripts/generate_sheet.py | 6 +- scripts/get_untriaged.py | 4 +- scripts/shadow_scheduler_stats.py | 4 +- scripts/test_scheduling_history_retriever.py | 6 +- scripts/testing_policy_stats.py | 7 ++- tests/test_repository.py | 6 +- 16 files changed, 102 insertions(+), 67 deletions(-) diff --git a/bugbug/bugzilla.py b/bugbug/bugzilla.py index fbef06fb70..8a1aee72a3 100644 --- a/bugbug/bugzilla.py +++ b/bugbug/bugzilla.py @@ -8,7 +8,7 @@ import itertools import math import re -from datetime import datetime +from datetime import datetime, timezone from logging import INFO, basicConfig, getLogger from typing import Iterable, Iterator, NewType from urllib.parse import urlencode @@ -360,7 +360,7 @@ def get_product_component_count(months: int = 12) -> dict[str, int]: `{product}::{component}`) and the value of the number of bugs for the given full components. Full component with 0 bugs are returned. """ - since = datetime.utcnow() - relativedelta(months=months) + since = datetime.now(timezone.utc) - relativedelta(months=months) # Base params params = { diff --git a/bugbug/models/accessibility.py b/bugbug/models/accessibility.py index e65644ee89..4516b36764 100644 --- a/bugbug/models/accessibility.py +++ b/bugbug/models/accessibility.py @@ -4,7 +4,7 @@ # You can obtain one at http://mozilla.org/MPL/2.0/. import logging -from datetime import datetime +from datetime import datetime, timezone import xgboost from dateutil.relativedelta import relativedelta @@ -94,7 +94,7 @@ def __download_older_access_bugs(months: int) -> None: This function provides an option to extend the dataset used for model training by including older bugs. """ - lookup_start_date = datetime.utcnow() - relativedelta(months=months) + lookup_start_date = datetime.now(timezone.utc) - relativedelta(months=months) params = { "f1": "creation_ts", "o1": "greaterthan", diff --git a/bugbug/models/backout.py b/bugbug/models/backout.py index 91900d459b..171c5ff327 100644 --- a/bugbug/models/backout.py +++ b/bugbug/models/backout.py @@ -4,10 +4,11 @@ # You can obtain one at http://mozilla.org/MPL/2.0/. import logging -from datetime import datetime +from datetime import datetime, timezone import dateutil.parser import xgboost +from dateutil import tz from dateutil.relativedelta import relativedelta from imblearn.pipeline import Pipeline as ImblearnPipeline from imblearn.under_sampling import RandomUnderSampler @@ -116,12 +117,17 @@ def __init__(self, lemmatization=False, bug_data=False): def get_labels(self): classes = {} - two_years_and_six_months_ago = datetime.utcnow() - relativedelta( + two_years_and_six_months_ago = datetime.now(timezone.utc) - relativedelta( years=2, months=6 ) for commit_data in repository.get_commits(): pushdate = dateutil.parser.parse(commit_data["pushdate"]) + if pushdate.tzinfo is None: + pushdate = pushdate.replace(tzinfo=tz.UTC) + else: + pushdate = pushdate.astimezone(tz.UTC) + if pushdate < two_years_and_six_months_ago: continue diff --git a/bugbug/models/regressor.py b/bugbug/models/regressor.py index eec647bb25..18bd8486a8 100644 --- a/bugbug/models/regressor.py +++ b/bugbug/models/regressor.py @@ -5,7 +5,7 @@ import itertools import logging -from datetime import datetime +from datetime import datetime, timezone import dateutil.parser import numpy as np @@ -188,7 +188,9 @@ def get_labels(self): push_date = dateutil.parser.parse(commit_data["pushdate"]) # Skip commits used for the evaluation phase. - if push_date > datetime.utcnow() - relativedelta(months=EVALUATION_MONTHS): + if push_date > datetime.now(timezone.utc) - relativedelta( + months=EVALUATION_MONTHS + ): continue node = commit_data["node"] @@ -203,7 +205,7 @@ def get_labels(self): # In the future, we might want to re-evaluate this limit (e.g. extend ), but we # have to be careful (using too old patches might cause worse results as patch # characteristics evolve over time). - if push_date < datetime.utcnow() - relativedelta(years=2): + if push_date < datetime.now(timezone.utc) - relativedelta(years=2): continue # We remove the last 3 months, as there could be regressions which haven't been @@ -211,7 +213,7 @@ def get_labels(self): # time, more than 3 months seems overly conservative. # There will be some patches we currently add to the clean set and will later move # to the regressor set, but they are a very small subset. - if push_date > datetime.utcnow() - relativedelta(months=3): + if push_date > datetime.now(timezone.utc) - relativedelta(months=3): continue classes[node] = 0 @@ -267,7 +269,9 @@ def evaluation(self) -> None: push_date = dateutil.parser.parse(commit_data["pushdate"]) # Use the past two months of data (make sure it is not also used for training!). - if push_date < datetime.utcnow() - relativedelta(months=EVALUATION_MONTHS): + if push_date < datetime.now(timezone.utc) - relativedelta( + months=EVALUATION_MONTHS + ): continue commits.append(commit_data) diff --git a/bugbug/phabricator.py b/bugbug/phabricator.py index 14827081d3..1fbdf12b6c 100644 --- a/bugbug/phabricator.py +++ b/bugbug/phabricator.py @@ -5,7 +5,7 @@ import itertools import logging -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from typing import Collection, Iterator, NewType import tenacity @@ -298,9 +298,9 @@ def get_pending_review_time(rev: RevisionDict) -> timedelta | None: ) if last_exclusion_end_date is not None: - return datetime.utcnow() - last_exclusion_end_date + return datetime.now(timezone.utc) - last_exclusion_end_date else: - return datetime.utcnow() - creation_date + return datetime.now(timezone.utc) - creation_date def fetch_diff_from_url( diff --git a/bugbug/repository.py b/bugbug/repository.py index 6110f04aa3..7ab899b6a3 100644 --- a/bugbug/repository.py +++ b/bugbug/repository.py @@ -20,7 +20,7 @@ import subprocess import sys import threading -from datetime import datetime +from datetime import datetime, timezone from functools import lru_cache from typing import Collection, Iterable, Iterator, NewType, Set, Union @@ -903,9 +903,9 @@ def hg_log( assert b" " in rev[6] pushdate_timestamp = rev[6].split(b" ", 1)[0] if pushdate_timestamp != b"0": - pushdate = datetime.utcfromtimestamp(float(pushdate_timestamp)) + pushdate = datetime.fromtimestamp(float(pushdate_timestamp), timezone.utc) else: - pushdate = datetime.utcnow() + pushdate = datetime.now(timezone.utc) bug_id = int(rev[3].decode("ascii")) if rev[3] else None diff --git a/functions/sync-review-comments-db/models.py b/functions/sync-review-comments-db/models.py index f18302029b..4213d1dc5f 100644 --- a/functions/sync-review-comments-db/models.py +++ b/functions/sync-review-comments-db/models.py @@ -4,10 +4,17 @@ # You can obtain one at http://mozilla.org/MPL/2.0/. import enum -from datetime import datetime +from datetime import datetime, timedelta, timezone from typing import List, Optional -from sqlalchemy import ForeignKey, ScalarResult, UniqueConstraint, func, select +from sqlalchemy import ( + DateTime, + ForeignKey, + ScalarResult, + UniqueConstraint, + func, + select, +) from sqlalchemy.orm import ( DeclarativeBase, Mapped, @@ -73,14 +80,19 @@ class ReviewRequest(Base): sequence: Mapped[int] # pylint:disable=not-callable - created_at: Mapped[datetime] = mapped_column(server_default=func.now()) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) updated_at: Mapped[datetime] = mapped_column( - server_default=func.now(), server_onupdate=func.now() + DateTime(timezone=True), + nullable=False, + server_default=func.now(), + server_onupdate=func.now(), ) @property def is_recently_created(self): - return (datetime.utcnow() - self.created_at).total_seconds() < 240 + return (datetime.now(timezone.utc) - self.created_at) < timedelta(minutes=4) def has_evaluation(self, session: Session) -> bool: if self.status == DiffStatus.IGNORED: diff --git a/scripts/bug_retriever.py b/scripts/bug_retriever.py index 1c6da38f2c..6b25e4621b 100644 --- a/scripts/bug_retriever.py +++ b/scripts/bug_retriever.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- import argparse -from datetime import datetime +from datetime import datetime, timezone from logging import getLogger import dateutil.parser @@ -61,7 +61,7 @@ def retrieve_bugs(self, limit: int | None = None) -> None: changed_ids |= deleted_component_ids # Get IDs of bugs between (two years and six months ago) and now. - two_years_and_six_months_ago = datetime.utcnow() - relativedelta( + two_years_and_six_months_ago = datetime.now(timezone.utc) - relativedelta( years=2, months=6 ) logger.info("Retrieving bug IDs since %s", two_years_and_six_months_ago) @@ -121,7 +121,7 @@ def retrieve_bugs(self, limit: int | None = None) -> None: test_failure_bug_ids = [ item["bug_id"] for item in test_scheduling.get_failure_bugs( - two_years_and_six_months_ago, datetime.utcnow() + two_years_and_six_months_ago, datetime.now(timezone.utc) ) if item["bug_id"] is not None ] diff --git a/scripts/commit_classifier.py b/scripts/commit_classifier.py index 72ea4af76a..2b0f504b11 100644 --- a/scripts/commit_classifier.py +++ b/scripts/commit_classifier.py @@ -9,7 +9,7 @@ import pickle import re import subprocess -from datetime import datetime +from datetime import datetime, timezone from logging import INFO, basicConfig, getLogger from typing import cast @@ -719,7 +719,7 @@ def classify_methods(self, commit): # Get commit hash from 4 months before the analysis time. # The method-level analyzer needs 4 months of history. stop_hash = None - four_months_ago = datetime.utcnow() - relativedelta(months=4) + four_months_ago = datetime.now(timezone.utc) - relativedelta(months=4) for commit in repository.get_commits(): if dateutil.parser.parse(commit["pushdate"]) >= four_months_ago: stop_hash = tuple( diff --git a/scripts/generate_landings_risk_report.py b/scripts/generate_landings_risk_report.py index e062773c91..6728323ec1 100644 --- a/scripts/generate_landings_risk_report.py +++ b/scripts/generate_landings_risk_report.py @@ -333,7 +333,7 @@ def get_prev_bugs_stats( ) def get_landed_and_filed_since(self, days: int) -> list[int]: - since = datetime.utcnow() - timedelta(days=days) + since = datetime.now(timezone.utc) - timedelta(days=days) commits = [] last_commit_by_bug: dict[int, datetime] = {} @@ -385,7 +385,7 @@ def retrieve_test_info(self, days: int) -> dict[str, Any]: db.download(TEST_INFOS_DB) dates = [ - datetime.utcnow() - timedelta(days=day) + datetime.now(timezone.utc) - timedelta(days=day) for day in reversed(range(min(days, 90))) ] @@ -399,7 +399,9 @@ def retrieve_test_info(self, days: int) -> dict[str, Any]: date_str = date.strftime("%Y-%m-%d") # Gather the latest three days again, as the data might have changed. - if date_str in test_infos and date < datetime.utcnow() - timedelta(days=3): + if date_str in test_infos and date < datetime.now(timezone.utc) - timedelta( + days=3 + ): prev_skips = test_infos[date_str]["skips"] continue @@ -942,7 +944,9 @@ def notification(days: int) -> None: if "bugs" not in data: continue - if dateutil.parser.parse(day) < datetime.utcnow() - relativedelta(weeks=1): + if dateutil.parser.parse(day) < datetime.now(timezone.utc) - relativedelta( + weeks=1 + ): continue all_intermittent_failure_bugs.update(bug["id"] for bug in data["bugs"]) @@ -1044,12 +1048,12 @@ def get_tracking_info(bug): (revision["pending_review_time"], revision["id"]) ) - if fix_date is not None and fix_date > datetime.utcnow() - relativedelta( - months=1 - ): + if fix_date is not None and fix_date > datetime.now( + timezone.utc + ) - relativedelta(months=1): cur_team_data["month_changes"] += 1 - if fix_date > datetime.utcnow() - relativedelta(weeks=1): + if fix_date > datetime.now(timezone.utc) - relativedelta(weeks=1): cur_team_data["fix_times"].append( (fix_date - creation_date).total_seconds() / 86400 ) @@ -1062,9 +1066,13 @@ def get_tracking_info(bug): cur_team_data["low_risk_changes"] += 1 if bug["regression"]: - if creation_date > datetime.utcnow() - relativedelta(weeks=1): + if creation_date > datetime.now(timezone.utc) - relativedelta( + weeks=1 + ): cur_team_data["week_old_fixed_regressions"] += 1 - elif creation_date > datetime.utcnow() - relativedelta(months=1): + elif creation_date > datetime.now(timezone.utc) - relativedelta( + months=1 + ): cur_team_data["month_old_fixed_regressions"] += 1 else: cur_team_data["more_than_month_old_fixed_regressions"] += 1 @@ -1080,9 +1088,9 @@ def get_tracking_info(bug): continue # We don't care about old commits associated to newly fixed bugs (e.g. a tentative fix from a year ago). - if dateutil.parser.parse( - commit["date"] - ) < datetime.utcnow() - relativedelta(weeks=1): + if dateutil.parser.parse(commit["date"]) < datetime.now( + timezone.utc + ) - relativedelta(weeks=1): continue lines_added = 0 @@ -1100,7 +1108,7 @@ def get_tracking_info(bug): (lines_covered / lines_added, commit["rev_id"]) ) - elif fix_date > datetime.utcnow() - relativedelta(weeks=2): + elif fix_date > datetime.now(timezone.utc) - relativedelta(weeks=2): cur_team_data["prev_fix_times"].append( (fix_date - creation_date).total_seconds() / 86400 ) @@ -1114,9 +1122,9 @@ def get_tracking_info(bug): if revision["first_review_time"] is not None ] - if bug["regression"] and creation_date > datetime.utcnow() - relativedelta( - weeks=2 - ): + if bug["regression"] and creation_date > datetime.now( + timezone.utc + ) - relativedelta(weeks=2): cur_team_data["new_regressions"] += 1 if bug["team"] == "Compiler and Development Tools": print("New regression: {}".format(bug["id"])) @@ -1128,13 +1136,13 @@ def get_tracking_info(bug): elif bug["assignee"] is None: cur_team_data["unassigned_new_regressions"] += 1 - if creation_date > datetime.utcnow() - relativedelta(weeks=2): + if creation_date > datetime.now(timezone.utc) - relativedelta(weeks=2): if bug["regression"] and not bug["fixed"]: if bug["team"] == "Compiler and Development Tools": print("Unfixed regression: {}".format(bug["id"])) cur_team_data["unfixed_regressions"].append(bug) - if creation_date > datetime.utcnow() - relativedelta(days=days): + if creation_date > datetime.now(timezone.utc) - relativedelta(days=days): if bug["regression"] and not bug["fixed"]: cur_team_data["carryover_regressions"] += 1 @@ -1183,14 +1191,18 @@ def get_tracking_info(bug): if "bugs" not in data: continue - if dateutil.parser.parse(day) < datetime.utcnow() - relativedelta(weeks=1): + if dateutil.parser.parse(day) < datetime.now(timezone.utc) - relativedelta( + weeks=1 + ): continue for bug in data["bugs"]: cur_team_data["intermittent_failures"][bug["id"]] += bug["count"] - today = datetime.utcnow().strftime("%Y-%m-%d") - two_weeks_ago = (datetime.utcnow() - timedelta(days=14)).strftime("%Y-%m-%d") + today = datetime.now(timezone.utc).strftime("%Y-%m-%d") + two_weeks_ago = (datetime.now(timezone.utc) - timedelta(days=14)).strftime( + "%Y-%m-%d" + ) skips = 0 prev_skips = 0 for day, data in day_to_data.items(): @@ -1796,13 +1808,13 @@ def get_top_crashes(team: str, channel: str) -> str | None: def calculate_maintenance_effectiveness( period: relativedelta, ) -> dict[str, dict]: - start_date = datetime.utcnow() - period + start_date = datetime.now(timezone.utc) - period if team in super_teams: me_teams = super_teams[team] else: me_teams = [team] return bugzilla.calculate_maintenance_effectiveness_indicator( - me_teams, start_date, datetime.utcnow() + me_teams, start_date, datetime.now(timezone.utc) ) def format_maintenance_effectiveness(period: relativedelta) -> str: diff --git a/scripts/generate_sheet.py b/scripts/generate_sheet.py index 47fee46ce2..bcecc57740 100644 --- a/scripts/generate_sheet.py +++ b/scripts/generate_sheet.py @@ -3,7 +3,7 @@ import argparse import csv import os -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from logging import INFO, basicConfig, getLogger import numpy as np @@ -26,7 +26,7 @@ def generate_sheet(model_name: str, token: str, days: int, threshold: float) -> model = model_class.load(model_file_name) bugzilla.set_token(token) - bug_ids = bugzilla.get_ids_between(datetime.utcnow() - timedelta(days)) + bug_ids = bugzilla.get_ids_between(datetime.now(timezone.utc) - timedelta(days)) bugs = bugzilla.get(bug_ids) logger.info("Classifying %d bugs...", len(bugs)) @@ -55,7 +55,7 @@ def generate_sheet(model_name: str, token: str, days: int, threshold: float) -> with open( os.path.join( "sheets", - f"{model_name}-{datetime.utcnow().strftime('%Y-%m-%d')}-labels.csv", + f"{model_name}-{datetime.now(timezone.utc).strftime('%Y-%m-%d')}-labels.csv", ), "w", ) as f: diff --git a/scripts/get_untriaged.py b/scripts/get_untriaged.py index 84ec86d62e..5c7c680155 100644 --- a/scripts/get_untriaged.py +++ b/scripts/get_untriaged.py @@ -8,7 +8,7 @@ import json import os import sys -from datetime import date, datetime, timedelta +from datetime import date, datetime, timedelta, timezone from logging import INFO, basicConfig, getLogger basicConfig(level=INFO) @@ -88,7 +88,7 @@ def run_untriaged(untriaged_bugs): os.makedirs("sheets", exist_ok=True) class_name = model.__class__.__name__ - timestamp = datetime.utcnow().strftime("%Y-%m-%d") + timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%d") sheet_name = f"{class_name}-{timestamp}-labels.csv" with open(os.path.join("sheets", sheet_name), "w") as f: writer = csv.writer(f) diff --git a/scripts/shadow_scheduler_stats.py b/scripts/shadow_scheduler_stats.py index 68b0599d1a..b17fca17fa 100644 --- a/scripts/shadow_scheduler_stats.py +++ b/scripts/shadow_scheduler_stats.py @@ -8,7 +8,7 @@ import logging import time import traceback -from datetime import datetime +from datetime import datetime, timezone from typing import Any import matplotlib.pyplot as plt @@ -66,7 +66,7 @@ def go(months: int) -> None: "Already gathered statistics for %d pushes...", len(known_scheduler_stats) ) - to_date = datetime.utcnow() - relativedelta(days=3) + to_date = datetime.now(timezone.utc) - relativedelta(days=3) from_date = to_date - relativedelta(months=months) pushes = mozci.push.make_push_objects( from_date=from_date.strftime("%Y-%m-%d"), diff --git a/scripts/test_scheduling_history_retriever.py b/scripts/test_scheduling_history_retriever.py index 56bfab677d..f10e07e259 100644 --- a/scripts/test_scheduling_history_retriever.py +++ b/scripts/test_scheduling_history_retriever.py @@ -8,7 +8,7 @@ import math import os import traceback -from datetime import datetime +from datetime import datetime, timezone from logging import INFO, basicConfig, getLogger from typing import Any, Generator @@ -42,8 +42,8 @@ def generate_push_data( # We use the actual date instead of 'today-X' aliases to avoid mozci caching # this query. - from_date = datetime.utcnow() - relativedelta(months=from_months) - to_date = datetime.utcnow() - relativedelta(days=3) + from_date = datetime.now(timezone.utc) - relativedelta(months=from_months) + to_date = datetime.now(timezone.utc) - relativedelta(days=3) if granularity == "label": push_data_db = test_scheduling.PUSH_DATA_LABEL_DB diff --git a/scripts/testing_policy_stats.py b/scripts/testing_policy_stats.py index 699a238f96..40ff6e43b7 100644 --- a/scripts/testing_policy_stats.py +++ b/scripts/testing_policy_stats.py @@ -7,7 +7,7 @@ import collections import logging import os -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from typing import Collection, Iterable import dateutil.parser @@ -51,8 +51,9 @@ def __init__(self, repo_dir: str) -> None: def get_landed_since( self, days_start: int, days_end: int ) -> Collection[repository.CommitDict]: - since = datetime.utcnow() - timedelta(days=days_start) - until = datetime.utcnow() - timedelta(days=days_end) + now = datetime.now(timezone.utc) + since = now - timedelta(days=days_start) + until = now - timedelta(days=days_end) return [ commit diff --git a/tests/test_repository.py b/tests/test_repository.py index 57ed777e7f..c1d5847e5e 100644 --- a/tests/test_repository.py +++ b/tests/test_repository.py @@ -249,7 +249,7 @@ def test_hg_log(fake_hg_repo): add_file(hg, local, "file1", "1\n2\n3\n4\n5\n6\n7\n") revision1 = commit(hg, date=datetime(1991, 4, 16, tzinfo=timezone.utc)) - first_push_date = datetime.utcnow() + first_push_date = datetime.now(timezone.utc) hg.push(dest=bytes(remote, "ascii")) add_file(hg, local, "file2", "1\n2\n3\n4\n5\n6\n7\n") @@ -278,7 +278,7 @@ def test_hg_log(fake_hg_repo): # Wait one second, to have a different pushdate. time.sleep(1) - second_push_date = datetime.utcnow() + second_push_date = datetime.now(timezone.utc) hg.push(dest=bytes(remote, "ascii")) add_file(hg, local, "file3", "1\n2\n3\n4\n5\n6\n7\n") @@ -291,7 +291,7 @@ def test_hg_log(fake_hg_repo): # Wait one second, to have a different pushdate. time.sleep(1) - hg_log_date = datetime.utcnow() + hg_log_date = datetime.now(timezone.utc) commits = repository.hg_log(hg, revs) assert len(commits) == 6, "hg log should return six commits"