Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions api/src/scripts/populate_db_gtfs.py
Original file line number Diff line number Diff line change
Expand Up @@ -290,6 +290,12 @@ def populate_db(self, session: "Session", fetch_url: bool = True):
is_producer_url_unstable_from_csv = self.get_safe_boolean_value(row, "is_producer_url_unstable", None)
if is_producer_url_unstable_from_csv is not None:
feed.is_producer_url_unstable = is_producer_url_unstable_from_csv
# Boolean flag from the catalog CSV ("True"/"False"/empty). The column defaults to FALSE, so a
# new/unmarked feed is non-seasonal. An empty cell leaves the current DB value untouched, so a
# re-import never wipes an operator-set value; only an explicit True/False overwrites it.
is_seasonal_from_csv = self.get_safe_boolean_value(row, "is_seasonal", None)
if is_seasonal_from_csv is not None:
feed.seasonal = is_seasonal_from_csv
feed.authentication_type = str(int(float(self.get_safe_value(row, "urls.authentication_type", "0"))))
feed.authentication_info_url = self.get_safe_value(row, "urls.authentication_info", "")
feed.api_key_parameter_name = self.get_safe_value(row, "urls.api_key_parameter_name", "")
Expand Down
2 changes: 2 additions & 0 deletions api/src/shared/db_models/feed_impl.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ def from_orm(cls, feed_orm: FeedOrm | None) -> Feed | None:
feed.status = feed_orm.status
feed.official = feed_orm.official
feed.official_updated_at = feed_orm.official_updated_at
feed.seasonal = feed_orm.seasonal
feed.feed_name = feed_orm.feed_name
feed.related_links = [FeedRelatedLinkImpl.from_orm(related_link) for related_link in feed_orm.feedrelatedlinks]
feed.note = feed_orm.note
Expand Down Expand Up @@ -70,6 +71,7 @@ def to_orm_from_dict(cls, feed_dict: dict | None, db_session: Session | None = N
status=feed_dict.get("status"),
official=feed_dict.get("official"),
official_updated_at=feed_dict.get("official_updated_at"),
seasonal=feed_dict.get("seasonal"),
feed_name=feed_dict.get("feed_name"),
note=feed_dict.get("note"),
externalids=(
Expand Down
2 changes: 2 additions & 0 deletions api/src/shared/db_models/search_feed_item_result_impl.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ def from_orm_gtfs(cls, feed_search_row: t_feedsearch):
provider=feed_search_row.provider,
feed_name=feed_search_row.feed_name,
official=feed_search_row.official,
seasonal=feed_search_row.seasonal,
note=feed_search_row.note,
feed_contact_email=feed_search_row.feed_contact_email,
source_info=SourceInfo(
Expand Down Expand Up @@ -116,6 +117,7 @@ def from_orm_gtfs_rt(cls, feed_search_row):
provider=feed_search_row.provider,
feed_name=feed_search_row.feed_name,
official=feed_search_row.official,
seasonal=feed_search_row.seasonal,
note=feed_search_row.note,
feed_contact_email=feed_search_row.feed_contact_email,
source_info=SourceInfo(
Expand Down
16 changes: 10 additions & 6 deletions api/tests/integration/populate_tests/test_data/sources_test.csv
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,19 @@
# and see the effect.
# We want to make sure the is_official flag is handled properly in that case.
# Normally changing to True or False should change the original value but changing to an empty cell should not
mdb_source_id,data_type,entity_type,location.country_code,location.subdivision_name,location.municipality,provider,is_official,name,note,feed_contact_email,static_reference,urls.direct_download,urls.authentication_type,urls.authentication_info,urls.api_key_parameter_name,urls.latest,urls.license,location.bounding_box.minimum_latitude,location.bounding_box.maximum_latitude,location.bounding_box.minimum_longitude,location.bounding_box.maximum_longitude,location.bounding_box.extracted_on,status,features,redirect.id,redirect.comment,is_producer_url_unstable
# For mdb-40, change is_official from TRUE to empty. Should retain True# For mdb-40, change is_official from TRUE to empty. Should retain True
40,gtfs,,CA,Ontario,London,London Transit Commission,,,,croy@londontransit.ca,,http://www.londontransit.ca/gtfsfeed/google_transit.zip,0,,,https://storage.googleapis.com/storage/v1/b/mdb-latest/o/ca-ontario-london-transit-commission-gtfs-2.zip?alt=media,https://www.londontransit.ca/open-data/ltcs-open-data-terms-of-use/,42.905244,43.051188,-81.36311,-81.137591,2022-02-22T19:51:34+00:00,inactive,,,,
mdb_source_id,data_type,entity_type,location.country_code,location.subdivision_name,location.municipality,provider,is_official,name,note,feed_contact_email,static_reference,urls.direct_download,urls.authentication_type,urls.authentication_info,urls.api_key_parameter_name,urls.latest,urls.license,location.bounding_box.minimum_latitude,location.bounding_box.maximum_latitude,location.bounding_box.minimum_longitude,location.bounding_box.maximum_longitude,location.bounding_box.extracted_on,status,features,redirect.id,redirect.comment,is_producer_url_unstable,is_seasonal
# For mdb-40, change is_official from TRUE to empty. Should retain True.
# Also leave is_seasonal empty here: should retain the base TRUE value.
40,gtfs,,CA,Ontario,London,London Transit Commission,,,,croy@londontransit.ca,,http://www.londontransit.ca/gtfsfeed/google_transit.zip,0,,,https://storage.googleapis.com/storage/v1/b/mdb-latest/o/ca-ontario-london-transit-commission-gtfs-2.zip?alt=media,https://www.londontransit.ca/open-data/ltcs-open-data-terms-of-use/,42.905244,43.051188,-81.36311,-81.137591,2022-02-22T19:51:34+00:00,inactive,,,,,
# For mdb-50, change is_official from FALSE to empty. Should retain False.
# Also leave is_producer_url_unstable empty here: should retain the base FALSE value.
50,gtfs,,CA,Ontario,Barrie,ZBarrie Transit,,,,,,http://www.myridebarrie.ca/gtfs/Google_transit.zip,,,,https://storage.googleapis.com/storage/v1/b/mdb-latest/o/ca-ontario-barrie-transit-gtfs-3.zip?alt=media,https://www.barrie.ca/services-payments/transportation-parking/barrie-transit/barrie-gtfs,44.3218044,44.42020676,-79.74063237,-79.61089569,2022-03-01T22:43:25+00:00,deprecated,,40|mdb-702,Some|Comment,
# Also leave is_seasonal empty here: should retain the base FALSE value.
50,gtfs,,CA,Ontario,Barrie,ZBarrie Transit,,,,,,http://www.myridebarrie.ca/gtfs/Google_transit.zip,,,,https://storage.googleapis.com/storage/v1/b/mdb-latest/o/ca-ontario-barrie-transit-gtfs-3.zip?alt=media,https://www.barrie.ca/services-payments/transportation-parking/barrie-transit/barrie-gtfs,44.3218044,44.42020676,-79.74063237,-79.61089569,2022-03-01T22:43:25+00:00,deprecated,,40|mdb-702,Some|Comment,,
# For mdb-1562, change is_official from FALSE to TRUE. Should change to True.
# Also change is_producer_url_unstable from base FALSE to TRUE: should change to True.
1562,gtfs-rt,sa,CA,BC,Vancouver,Vancouver-Transit(éèàçíóúČ),TRUE,Realtime(ŘŤÜÎ),,,50,http://foo.org/google_transit.zip,0,,,,,,,,,,active,,10,,TRUE
# Also change is_seasonal from base FALSE to TRUE: should change to True.
1562,gtfs-rt,sa,CA,BC,Vancouver,Vancouver-Transit(éèàçíóúČ),TRUE,Realtime(ŘŤÜÎ),,,50,http://foo.org/google_transit.zip,0,,,,,,,,,,active,,10,,TRUE,TRUE
# For mdb-1563, change is_official from TRUE to FALSE. Should change to False.
# Also change is_producer_url_unstable from base TRUE to FALSE: should change to False.
1563,gtfs-rt,tu,US,SomeState,SomeCity,SomeCity Bus,FALSE,RT,,,mdb-50,http://bar.com,0,,,,,,,,,,inactive,,10,,FALSE
# Also change is_seasonal from base TRUE to FALSE: should change to False.
1563,gtfs-rt,tu,US,SomeState,SomeCity,SomeCity Bus,FALSE,RT,,,mdb-50,http://bar.com,0,,,,,,,,,,inactive,,10,,FALSE,FALSE
79 changes: 79 additions & 0 deletions api/tests/integration/populate_tests/test_populate.py
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,85 @@ def test_is_producer_url_unstable_defaults_null(client: TestClient, values):
assert json_response["source_info"].get("is_producer_url_unstable") is None, values["assert_fail_message"]


@pytest.mark.parametrize(
"values",
[
{
"feed_id": "mdb-40",
"expected_seasonal": True,
"assert_fail_message": "mdb-40, is_seasonal changed from TRUE to empty. Should retain True.",
},
{
"feed_id": "mdb-50",
"expected_seasonal": False,
"assert_fail_message": "mdb-50, is_seasonal changed from FALSE to empty. Should retain False.",
},
{
"feed_id": "mdb-1562",
"expected_seasonal": True,
"assert_fail_message": "mdb-1562, is_seasonal FALSE->TRUE should change to True.",
},
{
"feed_id": "mdb-1563",
"expected_seasonal": False,
"assert_fail_message": "mdb-1563, is_seasonal TRUE->FALSE should change to False.",
},
],
ids=[
"seasonal_change_true_to_empty",
"seasonal_change_false_to_empty",
"seasonal_change_false_to_true",
"seasonal_change_true_to_false",
],
)
def test_is_seasonal_overwrite(client: TestClient, values):
"""An empty CSV cell must retain the stored value; an explicit True/False must overwrite it."""
feed_id = values["feed_id"]
expected = values["expected_seasonal"]

response = client.request(
"GET",
"/v1/feeds/{id}".format(id=feed_id),
headers=authHeaders,
)

assert response.status_code == 200
json_response = response.json()
assert json_response["seasonal"] is expected, values["assert_fail_message"]


@pytest.mark.parametrize(
"values",
[
{
"feed_id": "mdb-702",
"assert_fail_message": "mdb-702 has an empty is_seasonal cell; must default to False.",
},
{
"feed_id": "mdb-1",
"assert_fail_message": "mdb-1's CSV has no is_seasonal column; must default to False.",
},
],
ids=[
"seasonal_empty_cell_defaults_false",
"seasonal_absent_column_defaults_false",
],
)
def test_is_seasonal_defaults_false(client: TestClient, values):
"""An empty cell or a missing column must default seasonal to False (never null)."""
feed_id = values["feed_id"]

response = client.request(
"GET",
"/v1/feeds/{id}".format(id=feed_id),
headers=authHeaders,
)

assert response.status_code == 200
json_response = response.json()
assert json_response["seasonal"] is False, values["assert_fail_message"]


def test_is_feed_reference_overwrite(client: TestClient):
feed_id = "mdb-1562"
response = client.request(
Expand Down
12 changes: 6 additions & 6 deletions api/tests/integration/test_data/sources_test.csv
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
mdb_source_id,data_type,entity_type,location.country_code,location.subdivision_name,location.municipality,provider,is_official,name,note,feed_contact_email,static_reference,urls.direct_download,urls.authentication_type,urls.authentication_info,urls.api_key_parameter_name,urls.latest,urls.license,location.bounding_box.minimum_latitude,location.bounding_box.maximum_latitude,location.bounding_box.minimum_longitude,location.bounding_box.maximum_longitude,location.bounding_box.extracted_on,status,features,redirect.id,redirect.comment,is_producer_url_unstable
40,gtfs,,CA,Ontario,London,London Transit Commission,TRUE,,,croy@londontransit.ca,,http://www.londontransit.ca/gtfsfeed/google_transit.zip,0,,,https://storage.googleapis.com/storage/v1/b/mdb-latest/o/ca-ontario-london-transit-commission-gtfs-2.zip?alt=media,https://www.londontransit.ca/open-data/ltcs-open-data-terms-of-use/,42.905244,43.051188,-81.36311,-81.137591,2022-02-22T19:51:34+00:00,inactive,,,,TRUE
50,gtfs,,CA,Ontario,Barrie,ZBarrie Transit,FALSE,,,,,http://www.myridebarrie.ca/gtfs/Google_transit.zip,,,,https://storage.googleapis.com/storage/v1/b/mdb-latest/o/ca-ontario-barrie-transit-gtfs-3.zip?alt=media,https://www.barrie.ca/services-payments/transportation-parking/barrie-transit/barrie-gtfs,44.3218044,44.42020676,-79.74063237,-79.61089569,2022-03-01T22:43:25+00:00,deprecated,,40|mdb-702,Some|Comment,FALSE
702,gtfs,,CA,[British Columbia,Whistler],BC Transit (Whistler Transit System),,,,,,http://whistler.mapstrat.com/current/google_transit.zip,,,,https://storage.googleapis.com/storage/v1/b/mdb-latest/o/ca-british-columbia-bc-transit-whistler-transit-system-gtfs-702.zip?alt=media,https://www.bctransit.com/open-data/terms-of-use,50.077122,50.159071,-123.043635,-122.926836,2022-03-16T22:05:05+00:00,development,,,,
1562,gtfs-rt,sa,CA,BC,Vancouver,Vancouver-Transit(éèàçíóúČ),True,Realtime(ŘŤÜÎ),,,40|60,http://foo.org/google_transit.zip,0,,,,,,,,,,active,,10,,FALSE
1563,gtfs-rt,tu,US,SomeState,SomeCity,SomeCity Bus,False,RT,,,mdb-50,http://bar.com,0,,,,,,,,,,inactive,,10,,TRUE
mdb_source_id,data_type,entity_type,location.country_code,location.subdivision_name,location.municipality,provider,is_official,name,note,feed_contact_email,static_reference,urls.direct_download,urls.authentication_type,urls.authentication_info,urls.api_key_parameter_name,urls.latest,urls.license,location.bounding_box.minimum_latitude,location.bounding_box.maximum_latitude,location.bounding_box.minimum_longitude,location.bounding_box.maximum_longitude,location.bounding_box.extracted_on,status,features,redirect.id,redirect.comment,is_producer_url_unstable,is_seasonal
40,gtfs,,CA,Ontario,London,London Transit Commission,TRUE,,,croy@londontransit.ca,,http://www.londontransit.ca/gtfsfeed/google_transit.zip,0,,,https://storage.googleapis.com/storage/v1/b/mdb-latest/o/ca-ontario-london-transit-commission-gtfs-2.zip?alt=media,https://www.londontransit.ca/open-data/ltcs-open-data-terms-of-use/,42.905244,43.051188,-81.36311,-81.137591,2022-02-22T19:51:34+00:00,inactive,,,,TRUE,TRUE
50,gtfs,,CA,Ontario,Barrie,ZBarrie Transit,FALSE,,,,,http://www.myridebarrie.ca/gtfs/Google_transit.zip,,,,https://storage.googleapis.com/storage/v1/b/mdb-latest/o/ca-ontario-barrie-transit-gtfs-3.zip?alt=media,https://www.barrie.ca/services-payments/transportation-parking/barrie-transit/barrie-gtfs,44.3218044,44.42020676,-79.74063237,-79.61089569,2022-03-01T22:43:25+00:00,deprecated,,40|mdb-702,Some|Comment,FALSE,FALSE
702,gtfs,,CA,[British Columbia,Whistler],BC Transit (Whistler Transit System),,,,,,http://whistler.mapstrat.com/current/google_transit.zip,,,,https://storage.googleapis.com/storage/v1/b/mdb-latest/o/ca-british-columbia-bc-transit-whistler-transit-system-gtfs-702.zip?alt=media,https://www.bctransit.com/open-data/terms-of-use,50.077122,50.159071,-123.043635,-122.926836,2022-03-16T22:05:05+00:00,development,,,,,
1562,gtfs-rt,sa,CA,BC,Vancouver,Vancouver-Transit(éèàçíóúČ),True,Realtime(ŘŤÜÎ),,,40|60,http://foo.org/google_transit.zip,0,,,,,,,,,,active,,10,,FALSE,FALSE
1563,gtfs-rt,tu,US,SomeState,SomeCity,SomeCity Bus,False,RT,,,mdb-50,http://bar.com,0,,,,,,,,,,inactive,,10,,TRUE,TRUE
8 changes: 7 additions & 1 deletion api/tests/unittest/models/test_basic_feed_impl.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
note="note",
producer_url="producer_url",
is_producer_url_unstable=True,
seasonal=True,
authentication_type="1",
authentication_info_url="authentication_info_url",
api_key_parameter_name="api_key_parameter_name",
Expand Down Expand Up @@ -105,6 +106,7 @@
api_key_parameter_name="api_key_parameter_name",
license_url="license_url",
),
seasonal=True,
redirects=[
RedirectImpl(
target_id="target_id",
Expand Down Expand Up @@ -147,12 +149,14 @@ def test_from_orm_empty_fields(self):
# Test all None values
# No error should be raised
# Resulting list must be empty and not None
empty_feed_orm = Feed()
# seasonal is a NOT NULL DEFAULT FALSE column, so a persisted feed always carries a boolean.
empty_feed_orm = Feed(seasonal=False)
expected_empty_feed = FeedImpl(
external_ids=[],
redirects=[],
source_info=SourceInfo(),
related_links=[],
seasonal=False,
)
empty_result = FeedImpl.from_orm(empty_feed_orm)
assert empty_result == expected_empty_feed
Expand Down Expand Up @@ -181,6 +185,7 @@ def test_to_orm_from_dict_full_payload(self):
"feed_contact_email": "contact@example.com",
"producer_url": "https://producer.example.com",
"is_producer_url_unstable": True,
"seasonal": True,
"authentication_type": 1, # should be converted to string
"authentication_info_url": "https://auth.example.com",
"api_key_parameter_name": "api_key",
Expand Down Expand Up @@ -215,6 +220,7 @@ def test_to_orm_from_dict_full_payload(self):
assert obj.feed_contact_email == "contact@example.com"
assert obj.producer_url == "https://producer.example.com"
assert obj.is_producer_url_unstable is True
assert obj.seasonal is True
# authentication_type coerced to string per implementation
assert obj.authentication_type == "1"
assert obj.authentication_info_url == "https://auth.example.com"
Expand Down
2 changes: 2 additions & 0 deletions api/tests/unittest/models/test_gbfs_feed_impl.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ def setUp(self):
created_at=datetime(2024, 1, 1, 10, 0, 0),
data_type="gbfs",
system_id="sys1",
seasonal=False,
operator_url="https://provider.com",
locations=[self.location_orm],
gbfsversions=[self.version_orm],
Expand Down Expand Up @@ -75,6 +76,7 @@ def test_from_orm_empty_fields(self):
id="feed2",
stable_id="feed_stable_2",
system_id=None,
seasonal=False,
operator_url=None,
locations=[],
gbfsversions=[],
Expand Down
2 changes: 2 additions & 0 deletions api/tests/unittest/models/test_gtfs_feed_impl.py
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,7 @@ def create_test_notice(notice_code: str, total_notices: int, severity: str):
status="active",
feed_contact_email="feed_contact_email",
provider="provider",
seasonal=False,
locations=[
Location(
id="id",
Expand Down Expand Up @@ -212,6 +213,7 @@ def test_from_orm_empty_fields(self):
status=None,
feed_contact_email=None,
provider=None,
seasonal=False,
locations=[],
externalids=[],
latest_dataset=None,
Expand Down
1 change: 1 addition & 0 deletions api/tests/unittest/models/test_gtfs_rt_feed_impl.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
stable_id="id",
data_type="gtfs_rt",
status="active",
seasonal=False,
externalids=[
Externalid(
associated_id="associated_id",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ def __init__(self, **kwargs):
status="active",
feed_name="feed_name",
official=None,
seasonal=True,
created_at=fake.date_time_this_month(),
note="note",
feed_contact_email="feed_contact_email",
Expand Down Expand Up @@ -80,6 +81,7 @@ def test_from_orm_gtfs(self):
external_ids=item.external_ids,
provider=item.provider,
feed_name=item.feed_name,
seasonal=item.seasonal,
note=item.note,
feed_contact_email=item.feed_contact_email,
source_info=SourceInfo(
Expand Down Expand Up @@ -127,6 +129,7 @@ def test_from_orm_gtfs_rt(self):
external_ids=item.external_ids,
provider=item.provider,
feed_name=item.feed_name,
seasonal=item.seasonal,
note=item.note,
feed_contact_email=item.feed_contact_email,
source_info=SourceInfo(
Expand Down
1 change: 1 addition & 0 deletions api/tests/unittest/test_feeds.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
stable_id="test_id",
data_type="gtfs",
status="active",
seasonal=False,
provider="test_provider",
feed_name="test_feed_name",
created_at=datetime.fromisoformat("2023-07-10T22:06:00+00:00"),
Expand Down
Loading
Loading