Initial commit
This commit is contained in:
jkortis committed
commit
1d235d30e7
58 files changed
+19693
No files matched your search
@@ -0,0 +1,5 @@
|
||||
"""Media Sorter E2E Benchmark Test Suite & Offline Runner package."""
|
||||
|
||||
from .benchmark_cases import BENCHMARK_CASES, CASES_BY_DOMAIN, CASES_BY_ID, BenchmarkCase
|
||||
|
||||
__all__ = ["BenchmarkCase", "BENCHMARK_CASES", "CASES_BY_ID", "CASES_BY_DOMAIN"]
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,899 @@
|
||||
{
|
||||
"metadata": {
|
||||
"timestamp": "2026-09-06T02:50:04.839136+00:00",
|
||||
"python_version": "3.14.4",
|
||||
"total_duration_ms": 112.93,
|
||||
"runner": "MediaSorter Offline E2E Benchmark Runner"
|
||||
},
|
||||
"summary": {
|
||||
"total_cases": 64,
|
||||
"passed_cases": 64,
|
||||
"failed_cases": 0,
|
||||
"pass_rate_pct": 100.0
|
||||
},
|
||||
"domains": {
|
||||
"Standard TV": {
|
||||
"total": 11,
|
||||
"passed": 11,
|
||||
"failed": 0,
|
||||
"pass_rate_pct": 100.0,
|
||||
"total_duration_ms": 19.434,
|
||||
"avg_duration_ms": 1.77
|
||||
},
|
||||
"Anime": {
|
||||
"total": 11,
|
||||
"passed": 11,
|
||||
"failed": 0,
|
||||
"pass_rate_pct": 100.0,
|
||||
"total_duration_ms": 18.826,
|
||||
"avg_duration_ms": 1.71
|
||||
},
|
||||
"Movies": {
|
||||
"total": 14,
|
||||
"passed": 14,
|
||||
"failed": 0,
|
||||
"pass_rate_pct": 100.0,
|
||||
"total_duration_ms": 24.04,
|
||||
"avg_duration_ms": 1.72
|
||||
},
|
||||
"Specials & Extras": {
|
||||
"total": 9,
|
||||
"passed": 9,
|
||||
"failed": 0,
|
||||
"pass_rate_pct": 100.0,
|
||||
"total_duration_ms": 17.233999999999998,
|
||||
"avg_duration_ms": 1.91
|
||||
},
|
||||
"Daily / Dated Shows": {
|
||||
"total": 9,
|
||||
"passed": 9,
|
||||
"failed": 0,
|
||||
"pass_rate_pct": 100.0,
|
||||
"total_duration_ms": 15.488000000000001,
|
||||
"avg_duration_ms": 1.72
|
||||
},
|
||||
"Messy & Complex": {
|
||||
"total": 10,
|
||||
"passed": 10,
|
||||
"failed": 0,
|
||||
"pass_rate_pct": 100.0,
|
||||
"total_duration_ms": 17.905,
|
||||
"avg_duration_ms": 1.79
|
||||
}
|
||||
},
|
||||
"failures": [],
|
||||
"all_results": [
|
||||
{
|
||||
"id": "TV-01",
|
||||
"domain": "Standard TV",
|
||||
"filename": "Breaking.Bad.S05E14.Ozymandias.1080p.BluRay.x264-ROVERS.mkv",
|
||||
"edge_case_type": "standard_sxxexx",
|
||||
"passed": true,
|
||||
"duration_ms": 2.445,
|
||||
"diffs": {},
|
||||
"actual_category": "tv",
|
||||
"actual_title": "Breaking Bad",
|
||||
"actual_destination_subpath": "TV Shows/Breaking Bad/Season 05/Breaking Bad - S05E14.mkv",
|
||||
"exception": null
|
||||
},
|
||||
{
|
||||
"id": "TV-02",
|
||||
"domain": "Standard TV",
|
||||
"filename": "Stranger.Things.S04E01-E02.Chapter.One.720p.WEB-DL.mkv",
|
||||
"edge_case_type": "multi_episode_hyphen_range",
|
||||
"passed": true,
|
||||
"duration_ms": 1.765,
|
||||
"diffs": {},
|
||||
"actual_category": "tv",
|
||||
"actual_title": "Stranger Things",
|
||||
"actual_destination_subpath": "TV Shows/Stranger Things/Season 04/Stranger Things - S04E01-E02.mkv",
|
||||
"exception": null
|
||||
},
|
||||
{
|
||||
"id": "TV-03",
|
||||
"domain": "Standard TV",
|
||||
"filename": "House.M.D.S03E01E02.1080p.mkv",
|
||||
"edge_case_type": "multi_episode_concatenated",
|
||||
"passed": true,
|
||||
"duration_ms": 1.716,
|
||||
"diffs": {},
|
||||
"actual_category": "tv",
|
||||
"actual_title": "House M D",
|
||||
"actual_destination_subpath": "TV Shows/House M D/Season 03/House M D - S03E01-E02.mkv",
|
||||
"exception": null
|
||||
},
|
||||
{
|
||||
"id": "TV-04",
|
||||
"domain": "Standard TV",
|
||||
"filename": "The.Wire.1x09.HDTV.mkv",
|
||||
"edge_case_type": "scene_season_x_episode",
|
||||
"passed": true,
|
||||
"duration_ms": 1.695,
|
||||
"diffs": {},
|
||||
"actual_category": "tv",
|
||||
"actual_title": "The Wire",
|
||||
"actual_destination_subpath": "TV Shows/The Wire/Season 01/The Wire - S01E09.mkv",
|
||||
"exception": null
|
||||
},
|
||||
{
|
||||
"id": "TV-05",
|
||||
"domain": "Standard TV",
|
||||
"filename": "The.Office.2x01-02.mkv",
|
||||
"edge_case_type": "scene_multi_episode_range",
|
||||
"passed": true,
|
||||
"duration_ms": 1.692,
|
||||
"diffs": {},
|
||||
"actual_category": "tv",
|
||||
"actual_title": "The Office",
|
||||
"actual_destination_subpath": "TV Shows/The Office/Season 02/The Office - S02E01-E02.mkv",
|
||||
"exception": null
|
||||
},
|
||||
{
|
||||
"id": "TV-06",
|
||||
"domain": "Standard TV",
|
||||
"filename": "Rome.Season.II.Episode.IV.mkv",
|
||||
"edge_case_type": "roman_numerals",
|
||||
"passed": true,
|
||||
"duration_ms": 1.68,
|
||||
"diffs": {},
|
||||
"actual_category": "tv",
|
||||
"actual_title": "Rome",
|
||||
"actual_destination_subpath": "TV Shows/Rome/Season 02/Rome - S02E04.mkv",
|
||||
"exception": null
|
||||
},
|
||||
{
|
||||
"id": "TV-07",
|
||||
"domain": "Standard TV",
|
||||
"filename": "Doctor Who Season 5 Episode 1 Eleventh Hour.mkv",
|
||||
"edge_case_type": "word_season_episode",
|
||||
"passed": true,
|
||||
"duration_ms": 1.705,
|
||||
"diffs": {},
|
||||
"actual_category": "tv",
|
||||
"actual_title": "Doctor Who",
|
||||
"actual_destination_subpath": "TV Shows/Doctor Who/Season 05/Doctor Who - S05E01.mkv",
|
||||
"exception": null
|
||||
},
|
||||
{
|
||||
"id": "TV-08",
|
||||
"domain": "Standard TV",
|
||||
"filename": "Succession.S02.Complete.1080p.WEB-DL.mkv",
|
||||
"edge_case_type": "season_pack_complete",
|
||||
"passed": true,
|
||||
"duration_ms": 1.688,
|
||||
"diffs": {},
|
||||
"actual_category": "tv",
|
||||
"actual_title": "Succession",
|
||||
"actual_destination_subpath": "TV Shows/Succession/Season 02/Succession - Season 02.mkv",
|
||||
"exception": null
|
||||
},
|
||||
{
|
||||
"id": "TV-09",
|
||||
"domain": "Standard TV",
|
||||
"filename": "Game.of.Thrones.S08E03.720p.HDTV.x264-AVS.mkv",
|
||||
"edge_case_type": "standard_sxxexx_scene_group",
|
||||
"passed": true,
|
||||
"duration_ms": 1.69,
|
||||
"diffs": {},
|
||||
"actual_category": "tv",
|
||||
"actual_title": "Game of Thrones",
|
||||
"actual_destination_subpath": "TV Shows/Game of Thrones/Season 08/Game of Thrones - S08E03.mkv",
|
||||
"exception": null
|
||||
},
|
||||
{
|
||||
"id": "TV-10",
|
||||
"domain": "Standard TV",
|
||||
"filename": "Chernobyl.S01E05.Vichnaya.Pamyat.1080p.mkv",
|
||||
"edge_case_type": "sxxexx_with_episode_title",
|
||||
"passed": true,
|
||||
"duration_ms": 1.675,
|
||||
"diffs": {},
|
||||
"actual_category": "tv",
|
||||
"actual_title": "Chernobyl",
|
||||
"actual_destination_subpath": "TV Shows/Chernobyl/Season 01/Chernobyl - S01E05.mkv",
|
||||
"exception": null
|
||||
},
|
||||
{
|
||||
"id": "TV-11",
|
||||
"domain": "Standard TV",
|
||||
"filename": "Friends.S06E15-E16.The.One.That.Could.Have.Been.mkv",
|
||||
"edge_case_type": "multi_episode_double_digit",
|
||||
"passed": true,
|
||||
"duration_ms": 1.683,
|
||||
"diffs": {},
|
||||
"actual_category": "tv",
|
||||
"actual_title": "Friends",
|
||||
"actual_destination_subpath": "TV Shows/Friends/Season 06/Friends - S06E15-E16.mkv",
|
||||
"exception": null
|
||||
},
|
||||
{
|
||||
"id": "ANIME-01",
|
||||
"domain": "Anime",
|
||||
"filename": "[SubsPlease] Frieren - Beyond Journey's End - 01 (1080p) [ABCD1234].mkv",
|
||||
"edge_case_type": "fansub_brackets_crc",
|
||||
"passed": true,
|
||||
"duration_ms": 1.762,
|
||||
"diffs": {},
|
||||
"actual_category": "anime",
|
||||
"actual_title": "Frieren - Beyond Journey's End",
|
||||
"actual_destination_subpath": "Anime/Frieren - Beyond Journey's End/Frieren - Beyond Journey's End - 01 [SubsPlease].mkv",
|
||||
"exception": null
|
||||
},
|
||||
{
|
||||
"id": "ANIME-02",
|
||||
"domain": "Anime",
|
||||
"filename": "[SubsPlease] 葬送のフリーレン - 12 (1080p) [98E7B1A2].mkv",
|
||||
"edge_case_type": "unicode_kanji_title",
|
||||
"passed": true,
|
||||
"duration_ms": 1.714,
|
||||
"diffs": {},
|
||||
"actual_category": "anime",
|
||||
"actual_title": "葬送のフリーレン",
|
||||
"actual_destination_subpath": "Anime/葬送のフリーレン/葬送のフリーレン - 12 [SubsPlease].mkv",
|
||||
"exception": null
|
||||
},
|
||||
{
|
||||
"id": "ANIME-03",
|
||||
"domain": "Anime",
|
||||
"filename": "[HorribleSubs] Fairy Tail (2014) - 176 [720p].mkv",
|
||||
"edge_case_type": "parenthesized_title_year",
|
||||
"passed": true,
|
||||
"duration_ms": 1.688,
|
||||
"diffs": {},
|
||||
"actual_category": "anime",
|
||||
"actual_title": "Fairy Tail (2014)",
|
||||
"actual_destination_subpath": "Anime/Fairy Tail (2014)/Fairy Tail (2014) - 176 [HorribleSubs].mkv",
|
||||
"exception": null
|
||||
},
|
||||
{
|
||||
"id": "ANIME-04",
|
||||
"domain": "Anime",
|
||||
"filename": "[Erai-raws] One Piece - 1088 [1080p].mkv",
|
||||
"edge_case_type": "four_digit_absolute_numbering",
|
||||
"passed": true,
|
||||
"duration_ms": 1.688,
|
||||
"diffs": {},
|
||||
"actual_category": "anime",
|
||||
"actual_title": "One Piece",
|
||||
"actual_destination_subpath": "Anime/One Piece/One Piece - 1088 [Erai-raws].mkv",
|
||||
"exception": null
|
||||
},
|
||||
{
|
||||
"id": "ANIME-05",
|
||||
"domain": "Anime",
|
||||
"filename": "[SubsPlease] Dungeon Meshi - 01-02 (1080p).mkv",
|
||||
"edge_case_type": "multi_episode_anime",
|
||||
"passed": true,
|
||||
"duration_ms": 1.7,
|
||||
"diffs": {},
|
||||
"actual_category": "anime",
|
||||
"actual_title": "Dungeon Meshi",
|
||||
"actual_destination_subpath": "Anime/Dungeon Meshi/Dungeon Meshi - 01-02 [SubsPlease].mkv",
|
||||
"exception": null
|
||||
},
|
||||
{
|
||||
"id": "ANIME-06",
|
||||
"domain": "Anime",
|
||||
"filename": "[TaigaSubs] Attack on Titan OVA - 01 [720p].mkv",
|
||||
"edge_case_type": "anime_ova",
|
||||
"passed": true,
|
||||
"duration_ms": 1.709,
|
||||
"diffs": {},
|
||||
"actual_category": "anime",
|
||||
"actual_title": "Attack on Titan OVA",
|
||||
"actual_destination_subpath": "Anime/Attack on Titan OVA/Attack on Titan OVA - 01 [TaigaSubs].mkv",
|
||||
"exception": null
|
||||
},
|
||||
{
|
||||
"id": "ANIME-07",
|
||||
"domain": "Anime",
|
||||
"filename": "Naruto Episode 207 The Supposed Sealed Ability.mkv",
|
||||
"edge_case_type": "standalone_episode_keyword",
|
||||
"passed": true,
|
||||
"duration_ms": 1.715,
|
||||
"diffs": {},
|
||||
"actual_category": "anime",
|
||||
"actual_title": "Naruto",
|
||||
"actual_destination_subpath": "Anime/Naruto/Naruto - 207.mkv",
|
||||
"exception": null
|
||||
},
|
||||
{
|
||||
"id": "ANIME-08",
|
||||
"domain": "Anime",
|
||||
"filename": "[Judas] Fate Stay Night - Heaven's Feel - I. Presage Flower [BD 1080p].mkv",
|
||||
"edge_case_type": "anime_movie_roman_numeral",
|
||||
"passed": true,
|
||||
"duration_ms": 1.754,
|
||||
"diffs": {},
|
||||
"actual_category": "anime",
|
||||
"actual_title": "Fate Stay Night - Heaven's Feel - I. Presage Flower",
|
||||
"actual_destination_subpath": "Anime/Fate Stay Night - Heaven's Feel - I. Presage Flower/Fate Stay Night - Heaven's Feel - I. Presage Flower [Judas].mkv",
|
||||
"exception": null
|
||||
},
|
||||
{
|
||||
"id": "ANIME-09",
|
||||
"domain": "Anime",
|
||||
"filename": "BLEACH - Sennen Kessen-hen - 27 [E89717B7].mkv",
|
||||
"edge_case_type": "no_group_fansub_crc",
|
||||
"passed": true,
|
||||
"duration_ms": 1.677,
|
||||
"diffs": {},
|
||||
"actual_category": "anime",
|
||||
"actual_title": "BLEACH - Sennen Kessen-hen",
|
||||
"actual_destination_subpath": "Anime/BLEACH - Sennen Kessen-hen/BLEACH - Sennen Kessen-hen - 27.mkv",
|
||||
"exception": null
|
||||
},
|
||||
{
|
||||
"id": "ANIME-10",
|
||||
"domain": "Anime",
|
||||
"filename": "[Erai-raws] Jujutsu Kaisen 2nd Season - 14 [1080p][Multiple Subtitle].mkv",
|
||||
"edge_case_type": "cour_season_title_tags",
|
||||
"passed": true,
|
||||
"duration_ms": 1.715,
|
||||
"diffs": {},
|
||||
"actual_category": "anime",
|
||||
"actual_title": "Jujutsu Kaisen 2nd Season",
|
||||
"actual_destination_subpath": "Anime/Jujutsu Kaisen 2nd Season/Jujutsu Kaisen 2nd Season - 14 [Erai-raws].mkv",
|
||||
"exception": null
|
||||
},
|
||||
{
|
||||
"id": "ANIME-11",
|
||||
"domain": "Anime",
|
||||
"filename": "[SubsPlease] Mushoku Tensei S2 - 18 (1080p) [F28B1452].mkv",
|
||||
"edge_case_type": "short_season_notation",
|
||||
"passed": true,
|
||||
"duration_ms": 1.704,
|
||||
"diffs": {},
|
||||
"actual_category": "anime",
|
||||
"actual_title": "Mushoku Tensei S2",
|
||||
"actual_destination_subpath": "Anime/Mushoku Tensei S2/Mushoku Tensei S2 - 18 [SubsPlease].mkv",
|
||||
"exception": null
|
||||
},
|
||||
{
|
||||
"id": "MOVIE-01",
|
||||
"domain": "Movies",
|
||||
"filename": "Inception.2010.1080p.BluRay.x264-FraMeSToR.mkv",
|
||||
"edge_case_type": "standard_movie_year",
|
||||
"passed": true,
|
||||
"duration_ms": 1.792,
|
||||
"diffs": {},
|
||||
"actual_category": "movie",
|
||||
"actual_title": "Inception",
|
||||
"actual_destination_subpath": "Movies/Inception (2010)/Inception (2010).mkv",
|
||||
"exception": null
|
||||
},
|
||||
{
|
||||
"id": "MOVIE-02",
|
||||
"domain": "Movies",
|
||||
"filename": "1917.2019.1080p.BluRay.x264.mkv",
|
||||
"edge_case_type": "numerical_title_with_year",
|
||||
"passed": true,
|
||||
"duration_ms": 1.705,
|
||||
"diffs": {},
|
||||
"actual_category": "movie",
|
||||
"actual_title": "1917",
|
||||
"actual_destination_subpath": "Movies/1917 (2019)/1917 (2019).mkv",
|
||||
"exception": null
|
||||
},
|
||||
{
|
||||
"id": "MOVIE-03",
|
||||
"domain": "Movies",
|
||||
"filename": "2001.A.Space.Odyssey.1968.REMASTERED.1080p.mkv",
|
||||
"edge_case_type": "numerical_title_year_and_edition",
|
||||
"passed": true,
|
||||
"duration_ms": 1.72,
|
||||
"diffs": {},
|
||||
"actual_category": "movie",
|
||||
"actual_title": "2001 A Space Odyssey",
|
||||
"actual_destination_subpath": "Movies/2001 A Space Odyssey (1968)/2001 A Space Odyssey (1968) [Remastered].mkv",
|
||||
"exception": null
|
||||
},
|
||||
{
|
||||
"id": "MOVIE-04",
|
||||
"domain": "Movies",
|
||||
"filename": "Blade.Runner.2049.2017.2160p.UHD.BluRay.x265.mkv",
|
||||
"edge_case_type": "future_year_in_title",
|
||||
"passed": true,
|
||||
"duration_ms": 1.703,
|
||||
"diffs": {},
|
||||
"actual_category": "movie",
|
||||
"actual_title": "Blade Runner 2049",
|
||||
"actual_destination_subpath": "Movies/Blade Runner 2049 (2017)/Blade Runner 2049 (2017).mkv",
|
||||
"exception": null
|
||||
},
|
||||
{
|
||||
"id": "MOVIE-05",
|
||||
"domain": "Movies",
|
||||
"filename": "Wonder.Woman.1984.2020.1080p.WEB-DL.mkv",
|
||||
"edge_case_type": "year_in_title",
|
||||
"passed": true,
|
||||
"duration_ms": 1.718,
|
||||
"diffs": {},
|
||||
"actual_category": "movie",
|
||||
"actual_title": "Wonder Woman 1984",
|
||||
"actual_destination_subpath": "Movies/Wonder Woman 1984 (2020)/Wonder Woman 1984 (2020).mkv",
|
||||
"exception": null
|
||||
},
|
||||
{
|
||||
"id": "MOVIE-06",
|
||||
"domain": "Movies",
|
||||
"filename": "Class.of.1999.1990.720p.mkv",
|
||||
"edge_case_type": "past_year_in_title",
|
||||
"passed": true,
|
||||
"duration_ms": 1.703,
|
||||
"diffs": {},
|
||||
"actual_category": "movie",
|
||||
"actual_title": "Class of 1999",
|
||||
"actual_destination_subpath": "Movies/Class of 1999 (1990)/Class of 1999 (1990).mkv",
|
||||
"exception": null
|
||||
},
|
||||
{
|
||||
"id": "MOVIE-07",
|
||||
"domain": "Movies",
|
||||
"filename": "Titanic.1997.DVD.CD1.avi",
|
||||
"edge_case_type": "multi_part_cd1",
|
||||
"passed": true,
|
||||
"duration_ms": 1.696,
|
||||
"diffs": {},
|
||||
"actual_category": "movie",
|
||||
"actual_title": "Titanic",
|
||||
"actual_destination_subpath": "Movies/Titanic (1997)/Titanic (1997) [Pt.1].avi",
|
||||
"exception": null
|
||||
},
|
||||
{
|
||||
"id": "MOVIE-08",
|
||||
"domain": "Movies",
|
||||
"filename": "Titanic.1997.DVD.CD2.avi",
|
||||
"edge_case_type": "multi_part_cd2",
|
||||
"passed": true,
|
||||
"duration_ms": 1.692,
|
||||
"diffs": {},
|
||||
"actual_category": "movie",
|
||||
"actual_title": "Titanic",
|
||||
"actual_destination_subpath": "Movies/Titanic (1997)/Titanic (1997) [Pt.2].avi",
|
||||
"exception": null
|
||||
},
|
||||
{
|
||||
"id": "MOVIE-09",
|
||||
"domain": "Movies",
|
||||
"filename": "Kill.Bill.Vol.1.2003.1080p.BluRay.mkv",
|
||||
"edge_case_type": "volume_in_movie_title",
|
||||
"passed": true,
|
||||
"duration_ms": 1.71,
|
||||
"diffs": {},
|
||||
"actual_category": "movie",
|
||||
"actual_title": "Kill Bill Vol 1",
|
||||
"actual_destination_subpath": "Movies/Kill Bill Vol 1 (2003)/Kill Bill Vol 1 (2003).mkv",
|
||||
"exception": null
|
||||
},
|
||||
{
|
||||
"id": "MOVIE-10",
|
||||
"domain": "Movies",
|
||||
"filename": "The.Lord.of.the.Rings.The.Fellowship.of.the.Ring.2001.Extended.1080p.mkv",
|
||||
"edge_case_type": "edition_extended",
|
||||
"passed": true,
|
||||
"duration_ms": 1.713,
|
||||
"diffs": {},
|
||||
"actual_category": "movie",
|
||||
"actual_title": "The Lord of the Rings The Fellowship of the Ring",
|
||||
"actual_destination_subpath": "Movies/The Lord of the Rings The Fellowship of the Ring (2001)/The Lord of the Rings The Fellowship of the Ring (2001) [Extended].mkv",
|
||||
"exception": null
|
||||
},
|
||||
{
|
||||
"id": "MOVIE-11",
|
||||
"domain": "Movies",
|
||||
"filename": "Aliens.1986.Directors.Cut.1080p.BluRay.mkv",
|
||||
"edge_case_type": "edition_directors_cut",
|
||||
"passed": true,
|
||||
"duration_ms": 1.689,
|
||||
"diffs": {},
|
||||
"actual_category": "movie",
|
||||
"actual_title": "Aliens",
|
||||
"actual_destination_subpath": "Movies/Aliens (1986)/Aliens (1986) [Director's Cut].mkv",
|
||||
"exception": null
|
||||
},
|
||||
{
|
||||
"id": "MOVIE-12",
|
||||
"domain": "Movies",
|
||||
"filename": "Gladiator.2000.Remastered.1080p.BluRay.mkv",
|
||||
"edge_case_type": "edition_remastered",
|
||||
"passed": true,
|
||||
"duration_ms": 1.709,
|
||||
"diffs": {},
|
||||
"actual_category": "movie",
|
||||
"actual_title": "Gladiator",
|
||||
"actual_destination_subpath": "Movies/Gladiator (2000)/Gladiator (2000) [Remastered].mkv",
|
||||
"exception": null
|
||||
},
|
||||
{
|
||||
"id": "MOVIE-13",
|
||||
"domain": "Movies",
|
||||
"filename": "Seven.Samurai.1954.Criterion.Collection.1080p.BluRay.mkv",
|
||||
"edge_case_type": "edition_criterion",
|
||||
"passed": true,
|
||||
"duration_ms": 1.723,
|
||||
"diffs": {},
|
||||
"actual_category": "movie",
|
||||
"actual_title": "Seven Samurai",
|
||||
"actual_destination_subpath": "Movies/Seven Samurai (1954)/Seven Samurai (1954) [Criterion].mkv",
|
||||
"exception": null
|
||||
},
|
||||
{
|
||||
"id": "MOVIE-14",
|
||||
"domain": "Movies",
|
||||
"filename": "Blade.Runner.1982.Final.Cut.2160p.UHD.mkv",
|
||||
"edge_case_type": "edition_final_cut",
|
||||
"passed": true,
|
||||
"duration_ms": 1.767,
|
||||
"diffs": {},
|
||||
"actual_category": "movie",
|
||||
"actual_title": "Blade Runner",
|
||||
"actual_destination_subpath": "Movies/Blade Runner (1982)/Blade Runner (1982) [Final Cut].mkv",
|
||||
"exception": null
|
||||
},
|
||||
{
|
||||
"id": "SPECIAL-01",
|
||||
"domain": "Specials & Extras",
|
||||
"filename": "The.Office.S00E01.The.Outtakes.mkv",
|
||||
"edge_case_type": "season_00_special",
|
||||
"passed": true,
|
||||
"duration_ms": 3.02,
|
||||
"diffs": {},
|
||||
"actual_category": "tv",
|
||||
"actual_title": "The Office",
|
||||
"actual_destination_subpath": "TV Shows/The Office/Season 00/The Office - S00E01.mkv",
|
||||
"exception": null
|
||||
},
|
||||
{
|
||||
"id": "SPECIAL-02",
|
||||
"domain": "Specials & Extras",
|
||||
"filename": "Doctor.Who.S00E25.The.Day.of.the.Doctor.1080p.mkv",
|
||||
"edge_case_type": "season_00_high_episode",
|
||||
"passed": true,
|
||||
"duration_ms": 2.101,
|
||||
"diffs": {},
|
||||
"actual_category": "tv",
|
||||
"actual_title": "Doctor Who",
|
||||
"actual_destination_subpath": "TV Shows/Doctor Who/Season 00/Doctor Who - S00E25.mkv",
|
||||
"exception": null
|
||||
},
|
||||
{
|
||||
"id": "SPECIAL-03",
|
||||
"domain": "Specials & Extras",
|
||||
"filename": "Breaking.Bad.S05E00.Special.mkv",
|
||||
"edge_case_type": "mid_season_special_e00",
|
||||
"passed": true,
|
||||
"duration_ms": 1.927,
|
||||
"diffs": {},
|
||||
"actual_category": "tv",
|
||||
"actual_title": "Breaking Bad",
|
||||
"actual_destination_subpath": "TV Shows/Breaking Bad/Season 05/Breaking Bad - S05E00.mkv",
|
||||
"exception": null
|
||||
},
|
||||
{
|
||||
"id": "SPECIAL-04",
|
||||
"domain": "Specials & Extras",
|
||||
"filename": "Inception.2010-behindthescenes.mkv",
|
||||
"edge_case_type": "movie_extra_behind_the_scenes",
|
||||
"passed": true,
|
||||
"duration_ms": 1.712,
|
||||
"diffs": {},
|
||||
"actual_category": "movie",
|
||||
"actual_title": "Inception",
|
||||
"actual_destination_subpath": "Movies/Inception (2010)/Inception (2010)-behindthescenes.mkv",
|
||||
"exception": null
|
||||
},
|
||||
{
|
||||
"id": "SPECIAL-05",
|
||||
"domain": "Specials & Extras",
|
||||
"filename": "The.Matrix.1999-featurette.mkv",
|
||||
"edge_case_type": "movie_extra_featurette",
|
||||
"passed": true,
|
||||
"duration_ms": 1.697,
|
||||
"diffs": {},
|
||||
"actual_category": "movie",
|
||||
"actual_title": "The Matrix",
|
||||
"actual_destination_subpath": "Movies/The Matrix (1999)/The Matrix (1999)-featurette.mkv",
|
||||
"exception": null
|
||||
},
|
||||
{
|
||||
"id": "SPECIAL-06",
|
||||
"domain": "Specials & Extras",
|
||||
"filename": "Interstellar.2014-deleted.mkv",
|
||||
"edge_case_type": "movie_extra_deleted_scenes",
|
||||
"passed": true,
|
||||
"duration_ms": 1.684,
|
||||
"diffs": {},
|
||||
"actual_category": "movie",
|
||||
"actual_title": "Interstellar",
|
||||
"actual_destination_subpath": "Movies/Interstellar (2014)/Interstellar (2014)-deleted.mkv",
|
||||
"exception": null
|
||||
},
|
||||
{
|
||||
"id": "SPECIAL-07",
|
||||
"domain": "Specials & Extras",
|
||||
"filename": "Interstellar.2014-trailer.mp4",
|
||||
"edge_case_type": "movie_extra_trailer",
|
||||
"passed": true,
|
||||
"duration_ms": 1.684,
|
||||
"diffs": {},
|
||||
"actual_category": "movie",
|
||||
"actual_title": "Interstellar",
|
||||
"actual_destination_subpath": "Movies/Interstellar (2014)/Interstellar (2014)-trailer.mp4",
|
||||
"exception": null
|
||||
},
|
||||
{
|
||||
"id": "SPECIAL-08",
|
||||
"domain": "Specials & Extras",
|
||||
"filename": "Game.of.Thrones.S00E02.A.Day.in.the.Life.mkv",
|
||||
"edge_case_type": "tv_special_episode_title",
|
||||
"passed": true,
|
||||
"duration_ms": 1.701,
|
||||
"diffs": {},
|
||||
"actual_category": "tv",
|
||||
"actual_title": "Game of Thrones",
|
||||
"actual_destination_subpath": "TV Shows/Game of Thrones/Season 00/Game of Thrones - S00E02.mkv",
|
||||
"exception": null
|
||||
},
|
||||
{
|
||||
"id": "SPECIAL-09",
|
||||
"domain": "Specials & Extras",
|
||||
"filename": "Sherlock.S00E01.Many.Happy.Returns.mkv",
|
||||
"edge_case_type": "tv_special_prequel",
|
||||
"passed": true,
|
||||
"duration_ms": 1.708,
|
||||
"diffs": {},
|
||||
"actual_category": "tv",
|
||||
"actual_title": "Sherlock",
|
||||
"actual_destination_subpath": "TV Shows/Sherlock/Season 00/Sherlock - S00E01.mkv",
|
||||
"exception": null
|
||||
},
|
||||
{
|
||||
"id": "DAILY-01",
|
||||
"domain": "Daily / Dated Shows",
|
||||
"filename": "The.Daily.Show.2024-01-15.1080p.HDTV.mkv",
|
||||
"edge_case_type": "iso_dated_tv",
|
||||
"passed": true,
|
||||
"duration_ms": 1.711,
|
||||
"diffs": {},
|
||||
"actual_category": "tv",
|
||||
"actual_title": "The Daily Show",
|
||||
"actual_destination_subpath": "TV Shows/The Daily Show/Season 2024/The Daily Show - 2024-01-15.mkv",
|
||||
"exception": null
|
||||
},
|
||||
{
|
||||
"id": "DAILY-02",
|
||||
"domain": "Daily / Dated Shows",
|
||||
"filename": "The.Tonight.Show.Starring.Jimmy.Fallon.2024.03.12.720p.mkv",
|
||||
"edge_case_type": "dot_dated_tv",
|
||||
"passed": true,
|
||||
"duration_ms": 1.761,
|
||||
"diffs": {},
|
||||
"actual_category": "tv",
|
||||
"actual_title": "The Tonight Show Starring Jimmy Fallon",
|
||||
"actual_destination_subpath": "TV Shows/The Tonight Show Starring Jimmy Fallon/Season 2024/The Tonight Show Starring Jimmy Fallon - 2024-03-12.mkv",
|
||||
"exception": null
|
||||
},
|
||||
{
|
||||
"id": "DAILY-03",
|
||||
"domain": "Daily / Dated Shows",
|
||||
"filename": "Last.Week.Tonight.with.John.Oliver.2023-11-05.1080p.mkv",
|
||||
"edge_case_type": "weekly_dated_show",
|
||||
"passed": true,
|
||||
"duration_ms": 1.726,
|
||||
"diffs": {},
|
||||
"actual_category": "tv",
|
||||
"actual_title": "Last Week Tonight with John Oliver",
|
||||
"actual_destination_subpath": "TV Shows/Last Week Tonight with John Oliver/Season 2023/Last Week Tonight with John Oliver - 2023-11-05.mkv",
|
||||
"exception": null
|
||||
},
|
||||
{
|
||||
"id": "DAILY-04",
|
||||
"domain": "Daily / Dated Shows",
|
||||
"filename": "Late.Night.with.Seth.Meyers.2024_02_20.720p.mkv",
|
||||
"edge_case_type": "underscore_dated_tv",
|
||||
"passed": true,
|
||||
"duration_ms": 1.725,
|
||||
"diffs": {},
|
||||
"actual_category": "tv",
|
||||
"actual_title": "Late Night with Seth Meyers",
|
||||
"actual_destination_subpath": "TV Shows/Late Night with Seth Meyers/Season 2024/Late Night with Seth Meyers - 2024-02-20.mkv",
|
||||
"exception": null
|
||||
},
|
||||
{
|
||||
"id": "DAILY-05",
|
||||
"domain": "Daily / Dated Shows",
|
||||
"filename": "The Daily - 2026-03-12 - The Sunday Read.mp3",
|
||||
"edge_case_type": "dated_podcast",
|
||||
"passed": true,
|
||||
"duration_ms": 1.711,
|
||||
"diffs": {},
|
||||
"actual_category": "podcast",
|
||||
"actual_title": "The Sunday Read",
|
||||
"actual_destination_subpath": "Podcasts/The Daily/2026/The Daily - 2026-03-12 - The Sunday Read.mp3",
|
||||
"exception": null
|
||||
},
|
||||
{
|
||||
"id": "DAILY-06",
|
||||
"domain": "Daily / Dated Shows",
|
||||
"filename": "Jimmy.Kimmel.Live.2024-04-18.720p.HDTV.mkv",
|
||||
"edge_case_type": "daily_show_iso",
|
||||
"passed": true,
|
||||
"duration_ms": 1.713,
|
||||
"diffs": {},
|
||||
"actual_category": "tv",
|
||||
"actual_title": "Jimmy Kimmel Live",
|
||||
"actual_destination_subpath": "TV Shows/Jimmy Kimmel Live/Season 2024/Jimmy Kimmel Live - 2024-04-18.mkv",
|
||||
"exception": null
|
||||
},
|
||||
{
|
||||
"id": "DAILY-07",
|
||||
"domain": "Daily / Dated Shows",
|
||||
"filename": "PBS.NewsHour.2024.05.01.720p.mkv",
|
||||
"edge_case_type": "news_broadcast_dot_date",
|
||||
"passed": true,
|
||||
"duration_ms": 1.709,
|
||||
"diffs": {},
|
||||
"actual_category": "tv",
|
||||
"actual_title": "PBS NewsHour",
|
||||
"actual_destination_subpath": "TV Shows/PBS NewsHour/Season 2024/PBS NewsHour - 2024-05-01.mkv",
|
||||
"exception": null
|
||||
},
|
||||
{
|
||||
"id": "DAILY-08",
|
||||
"domain": "Daily / Dated Shows",
|
||||
"filename": "The.Late.Show.with.Stephen.Colbert.2024-02-14.1080p.mkv",
|
||||
"edge_case_type": "late_night_show_iso",
|
||||
"passed": true,
|
||||
"duration_ms": 1.724,
|
||||
"diffs": {},
|
||||
"actual_category": "tv",
|
||||
"actual_title": "The Late Show with Stephen Colbert",
|
||||
"actual_destination_subpath": "TV Shows/The Late Show with Stephen Colbert/Season 2024/The Late Show with Stephen Colbert - 2024-02-14.mkv",
|
||||
"exception": null
|
||||
},
|
||||
{
|
||||
"id": "DAILY-09",
|
||||
"domain": "Daily / Dated Shows",
|
||||
"filename": "NPR.News.Now.2024-06-10.mp3",
|
||||
"edge_case_type": "podcast_daily_news",
|
||||
"passed": true,
|
||||
"duration_ms": 1.708,
|
||||
"diffs": {},
|
||||
"actual_category": "podcast",
|
||||
"actual_title": "NPR News Now",
|
||||
"actual_destination_subpath": "Podcasts/NPR News Now/2024/NPR News Now - 2024-06-10.mp3",
|
||||
"exception": null
|
||||
},
|
||||
{
|
||||
"id": "MESSY-01",
|
||||
"domain": "Messy & Complex",
|
||||
"filename": "Amélie.2001.PROPER.REMASTERED.1080p.BluRay.x264-CiNEFiLE.mkv",
|
||||
"edge_case_type": "unicode_accented_characters",
|
||||
"passed": true,
|
||||
"duration_ms": 1.764,
|
||||
"diffs": {},
|
||||
"actual_category": "movie",
|
||||
"actual_title": "Amélie",
|
||||
"actual_destination_subpath": "Movies/Amélie (2001)/Amélie (2001) [Remastered].mkv",
|
||||
"exception": null
|
||||
},
|
||||
{
|
||||
"id": "MESSY-02",
|
||||
"domain": "Messy & Complex",
|
||||
"filename": "Wolfs.2024.1080p.Apple.TV.WEB-DL.DDP5.1.Atmos.H.264.mkv",
|
||||
"edge_case_type": "apple_tv_scene_tag",
|
||||
"passed": true,
|
||||
"duration_ms": 1.758,
|
||||
"diffs": {},
|
||||
"actual_category": "movie",
|
||||
"actual_title": "Wolfs",
|
||||
"actual_destination_subpath": "Movies/Wolfs (2024)/Wolfs (2024).mkv",
|
||||
"exception": null
|
||||
},
|
||||
{
|
||||
"id": "MESSY-03",
|
||||
"domain": "Messy & Complex",
|
||||
"filename": "Gladiator.II.2024.1080p.HDTV.x264-[rartv].mkv",
|
||||
"edge_case_type": "roman_numeral_title_bracket_group",
|
||||
"passed": true,
|
||||
"duration_ms": 1.742,
|
||||
"diffs": {},
|
||||
"actual_category": "movie",
|
||||
"actual_title": "Gladiator II",
|
||||
"actual_destination_subpath": "Movies/Gladiator II (2024)/Gladiator II (2024).mkv",
|
||||
"exception": null
|
||||
},
|
||||
{
|
||||
"id": "MESSY-04",
|
||||
"domain": "Messy & Complex",
|
||||
"filename": "Interstellar.1920x1080.mkv",
|
||||
"edge_case_type": "resolution_dimensions",
|
||||
"passed": true,
|
||||
"duration_ms": 1.712,
|
||||
"diffs": {},
|
||||
"actual_category": "movie",
|
||||
"actual_title": "Interstellar",
|
||||
"actual_destination_subpath": "Movies/Interstellar/Interstellar.mkv",
|
||||
"exception": null
|
||||
},
|
||||
{
|
||||
"id": "MESSY-05",
|
||||
"domain": "Messy & Complex",
|
||||
"filename": "[YTS.MX] Movie Title - 2024 [1080p].mkv",
|
||||
"edge_case_type": "bracket_group_movie_not_anime",
|
||||
"passed": true,
|
||||
"duration_ms": 1.747,
|
||||
"diffs": {},
|
||||
"actual_category": "movie",
|
||||
"actual_title": "Movie Title",
|
||||
"actual_destination_subpath": "Movies/Movie Title (2024)/Movie Title (2024).mkv",
|
||||
"exception": null
|
||||
},
|
||||
{
|
||||
"id": "MESSY-06",
|
||||
"domain": "Messy & Complex",
|
||||
"filename": "The.Dark.Knight.2008.1080p.forced.srt",
|
||||
"edge_case_type": "subtitle_forced_tag",
|
||||
"passed": true,
|
||||
"duration_ms": 2.06,
|
||||
"diffs": {},
|
||||
"actual_category": "subtitle",
|
||||
"actual_title": "The Dark Knight",
|
||||
"actual_destination_subpath": "Movies/The Dark Knight (2008)/The Dark Knight (2008).forced.srt",
|
||||
"exception": null
|
||||
},
|
||||
{
|
||||
"id": "MESSY-07",
|
||||
"domain": "Messy & Complex",
|
||||
"filename": "Show: \"Special\" <Episode> | 1?.mkv",
|
||||
"edge_case_type": "illegal_characters",
|
||||
"passed": true,
|
||||
"duration_ms": 1.807,
|
||||
"diffs": {},
|
||||
"actual_category": "tv",
|
||||
"actual_title": "Show Special Episode 1",
|
||||
"actual_destination_subpath": "TV Shows/Show Special Episode 1/Season 01/Show Special Episode 1 - S01E01.mkv",
|
||||
"exception": null
|
||||
},
|
||||
{
|
||||
"id": "MESSY-08",
|
||||
"domain": "Messy & Complex",
|
||||
"filename": "CON.mp4",
|
||||
"edge_case_type": "windows_reserved_device_name",
|
||||
"passed": true,
|
||||
"duration_ms": 1.845,
|
||||
"diffs": {},
|
||||
"actual_category": "home_video",
|
||||
"actual_title": "CON",
|
||||
"actual_destination_subpath": "Home Videos/2026/2026-01 - Event/_CON.mp4",
|
||||
"exception": null
|
||||
},
|
||||
{
|
||||
"id": "MESSY-09",
|
||||
"domain": "Messy & Complex",
|
||||
"filename": " Messy Show . S01E01 . 1080p .mkv",
|
||||
"edge_case_type": "irregular_whitespace_and_dots",
|
||||
"passed": true,
|
||||
"duration_ms": 1.738,
|
||||
"diffs": {},
|
||||
"actual_category": "tv",
|
||||
"actual_title": "Messy Show",
|
||||
"actual_destination_subpath": "TV Shows/Messy Show/Season 01/Messy Show - S01E01.mkv",
|
||||
"exception": null
|
||||
},
|
||||
{
|
||||
"id": "MESSY-10",
|
||||
"domain": "Messy & Complex",
|
||||
"filename": "Show_Name__2022__S02E03__HDTV.mkv",
|
||||
"edge_case_type": "consecutive_underscores",
|
||||
"passed": true,
|
||||
"duration_ms": 1.732,
|
||||
"diffs": {},
|
||||
"actual_category": "tv",
|
||||
"actual_title": "Show Name",
|
||||
"actual_destination_subpath": "TV Shows/Show Name/Season 02/Show Name - S02E03.mkv",
|
||||
"exception": null
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,344 @@
|
||||
"""Standalone CLI Benchmark Runner & Diagnostic Reporter (Requirement R2).
|
||||
|
||||
Executable via:
|
||||
.venv/bin/python -m tests.benchmark.runner
|
||||
.venv/bin/python tests/benchmark/runner.py
|
||||
|
||||
Computes per-domain pass/fail statistics, execution timing, and field diffs for
|
||||
any failed case. Renders a clean terminal summary table and exports
|
||||
tests/benchmark/benchmark_summary.json.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from datetime import datetime, timezone
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
# Ensure project root is in sys.path
|
||||
_PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
|
||||
if str(_PROJECT_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(_PROJECT_ROOT))
|
||||
|
||||
try:
|
||||
from tests.benchmark.benchmark_cases import (
|
||||
BENCHMARK_CASES,
|
||||
BenchmarkCase,
|
||||
CaseResult,
|
||||
evaluate_benchmark_case,
|
||||
)
|
||||
except ImportError:
|
||||
from benchmark_cases import ( # type: ignore
|
||||
BENCHMARK_CASES,
|
||||
BenchmarkCase,
|
||||
CaseResult,
|
||||
evaluate_benchmark_case,
|
||||
)
|
||||
|
||||
DEFAULT_JSON_PATH = Path(__file__).parent / "benchmark_summary.json"
|
||||
|
||||
|
||||
class BenchmarkRunner:
|
||||
"""Orchestrates execution, diagnostic collection, and reporting for benchmark cases."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
cases: Optional[List[BenchmarkCase]] = None,
|
||||
json_output_path: Path = DEFAULT_JSON_PATH,
|
||||
verbose: bool = False,
|
||||
show_diffs: bool = True,
|
||||
strict: bool = False,
|
||||
):
|
||||
self.cases = cases or BENCHMARK_CASES
|
||||
self.json_output_path = json_output_path
|
||||
self.verbose = verbose
|
||||
self.show_diffs = show_diffs
|
||||
self.strict = strict
|
||||
self.results: List[CaseResult] = []
|
||||
|
||||
def run(self) -> Dict[str, Any]:
|
||||
"""Execute all configured benchmark cases and collect results."""
|
||||
self.results.clear()
|
||||
for case in self.cases:
|
||||
result = evaluate_benchmark_case(case)
|
||||
self.results.append(result)
|
||||
|
||||
summary_data = self._build_summary_data()
|
||||
self._export_json(summary_data)
|
||||
return summary_data
|
||||
|
||||
def _build_summary_data(self) -> Dict[str, Any]:
|
||||
total_cases = len(self.results)
|
||||
passed_cases = sum(1 for r in self.results if r.passed)
|
||||
failed_cases = total_cases - passed_cases
|
||||
pass_rate = (passed_cases / total_cases * 100.0) if total_cases else 0.0
|
||||
total_duration_ms = sum(r.duration_ms for r in self.results)
|
||||
|
||||
# Domain breakdown
|
||||
domain_stats: Dict[str, Dict[str, Any]] = {}
|
||||
for r in self.results:
|
||||
d = domain_stats.setdefault(
|
||||
r.domain,
|
||||
{
|
||||
"total": 0,
|
||||
"passed": 0,
|
||||
"failed": 0,
|
||||
"pass_rate_pct": 0.0,
|
||||
"total_duration_ms": 0.0,
|
||||
"avg_duration_ms": 0.0,
|
||||
},
|
||||
)
|
||||
d["total"] += 1
|
||||
if r.passed:
|
||||
d["passed"] += 1
|
||||
else:
|
||||
d["failed"] += 1
|
||||
d["total_duration_ms"] += r.duration_ms
|
||||
|
||||
for d in domain_stats.values():
|
||||
if d["total"] > 0:
|
||||
d["pass_rate_pct"] = round(d["passed"] / d["total"] * 100.0, 1)
|
||||
d["avg_duration_ms"] = round(d["total_duration_ms"] / d["total"], 2)
|
||||
|
||||
# Failures list
|
||||
failures = [
|
||||
{
|
||||
"id": r.case_id,
|
||||
"domain": r.domain,
|
||||
"filename": r.filename,
|
||||
"edge_case_type": r.edge_case_type,
|
||||
"diffs": r.diffs,
|
||||
"actual_category": r.actual_category,
|
||||
"actual_title": r.actual_title,
|
||||
"actual_destination_subpath": r.actual_destination_subpath,
|
||||
"duration_ms": r.duration_ms,
|
||||
}
|
||||
for r in self.results
|
||||
if not r.passed
|
||||
]
|
||||
|
||||
# Serialized results
|
||||
all_results_data = [
|
||||
{
|
||||
"id": r.case_id,
|
||||
"domain": r.domain,
|
||||
"filename": r.filename,
|
||||
"edge_case_type": r.edge_case_type,
|
||||
"passed": r.passed,
|
||||
"duration_ms": r.duration_ms,
|
||||
"diffs": r.diffs,
|
||||
"actual_category": r.actual_category,
|
||||
"actual_title": r.actual_title,
|
||||
"actual_destination_subpath": r.actual_destination_subpath,
|
||||
"exception": r.exception,
|
||||
}
|
||||
for r in self.results
|
||||
]
|
||||
|
||||
return {
|
||||
"metadata": {
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"python_version": sys.version.split()[0],
|
||||
"total_duration_ms": round(total_duration_ms, 2),
|
||||
"runner": "MediaSorter Offline E2E Benchmark Runner",
|
||||
},
|
||||
"summary": {
|
||||
"total_cases": total_cases,
|
||||
"passed_cases": passed_cases,
|
||||
"failed_cases": failed_cases,
|
||||
"pass_rate_pct": round(pass_rate, 2),
|
||||
},
|
||||
"domains": domain_stats,
|
||||
"failures": failures,
|
||||
"all_results": all_results_data,
|
||||
}
|
||||
|
||||
def _export_json(self, data: Dict[str, Any]) -> None:
|
||||
self.json_output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(self.json_output_path, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, indent=2, ensure_ascii=False)
|
||||
|
||||
def print_terminal_report(self, summary_data: Dict[str, Any]) -> None:
|
||||
"""Render a styled terminal summary table and failure details."""
|
||||
try:
|
||||
self._print_rich_report(summary_data)
|
||||
except Exception:
|
||||
self._print_plain_report(summary_data)
|
||||
|
||||
def _print_rich_report(self, data: Dict[str, Any]) -> None:
|
||||
from rich.console import Console
|
||||
from rich.markup import escape
|
||||
from rich.panel import Panel
|
||||
from rich.table import Table
|
||||
|
||||
console = Console()
|
||||
console.print()
|
||||
console.print(
|
||||
Panel.fit(
|
||||
"[bold cyan]Media Sorter E2E Benchmark Test Suite & Offline Runner[/bold cyan]\n"
|
||||
f"[dim]Timestamp: {data['metadata']['timestamp']} | Total Cases: {data['summary']['total_cases']}[/dim]",
|
||||
border_style="cyan",
|
||||
)
|
||||
)
|
||||
|
||||
table = Table(title="Benchmark Results by Domain", show_footer=True)
|
||||
table.add_column("Domain", style="bold white", footer="Total / Overall")
|
||||
table.add_column("Total", justify="right", footer=str(data["summary"]["total_cases"]))
|
||||
table.add_column("Passed", justify="right", style="green", footer=str(data["summary"]["passed_cases"]))
|
||||
table.add_column("Failed", justify="right", style="red", footer=str(data["summary"]["failed_cases"]))
|
||||
table.add_column(
|
||||
"Pass Rate",
|
||||
justify="right",
|
||||
style="bold yellow",
|
||||
footer=f"{data['summary']['pass_rate_pct']:.1f}%",
|
||||
)
|
||||
table.add_column(
|
||||
"Avg Time (ms)",
|
||||
justify="right",
|
||||
style="dim",
|
||||
footer=f"{data['metadata']['total_duration_ms']:.1f}ms total",
|
||||
)
|
||||
|
||||
for domain, stats in data["domains"].items():
|
||||
rate = stats["pass_rate_pct"]
|
||||
rate_style = "green" if rate == 100.0 else ("yellow" if rate >= 50.0 else "red")
|
||||
table.add_row(
|
||||
domain,
|
||||
str(stats["total"]),
|
||||
str(stats["passed"]),
|
||||
str(stats["failed"]),
|
||||
f"[{rate_style}]{rate:.1f}%[/{rate_style}]",
|
||||
f"{stats['avg_duration_ms']:.2f}ms",
|
||||
)
|
||||
|
||||
console.print(table)
|
||||
console.print()
|
||||
|
||||
# Print failures if requested
|
||||
if self.show_diffs and data["failures"]:
|
||||
console.print(f"[bold red]Diagnostic Gap Details ({len(data['failures'])} failures pending M2/M3):[/bold red]")
|
||||
for item in data["failures"]:
|
||||
esc_filename = escape(item["filename"])
|
||||
console.print(
|
||||
f"\n [bold red]✖ [{item['id']}][/bold red] [bold white]{esc_filename}[/bold white] "
|
||||
f"([dim]{item['domain']} / {item['edge_case_type']}[/dim])"
|
||||
)
|
||||
for field_name, diff in item["diffs"].items():
|
||||
exp_val = escape(repr(diff["expected"]))
|
||||
act_val = escape(repr(diff["actual"]))
|
||||
console.print(
|
||||
f" [yellow]• {field_name}:[/yellow] "
|
||||
f"expected=[green]{exp_val}[/green], got=[red]{act_val}[/red]"
|
||||
)
|
||||
|
||||
|
||||
console.print()
|
||||
console.print(
|
||||
f"[dim]Summary report exported to:[/dim] [cyan]{self.json_output_path.resolve()}[/cyan]\n"
|
||||
)
|
||||
|
||||
def _print_plain_report(self, data: Dict[str, Any]) -> None:
|
||||
print("=" * 80)
|
||||
print(" Media Sorter E2E Benchmark Test Suite & Offline Runner")
|
||||
print(f" Timestamp: {data['metadata']['timestamp']} | Total: {data['summary']['total_cases']}")
|
||||
print("=" * 80)
|
||||
print(f"{'Domain':<25} {'Total':>8} {'Passed':>8} {'Failed':>8} {'Pass %':>10} {'Avg (ms)':>10}")
|
||||
print("-" * 80)
|
||||
for domain, stats in data["domains"].items():
|
||||
print(
|
||||
f"{domain:<25} {stats['total']:>8} {stats['passed']:>8} {stats['failed']:>8} "
|
||||
f"{stats['pass_rate_pct']:>9.1f}% {stats['avg_duration_ms']:>10.2f}"
|
||||
)
|
||||
print("-" * 80)
|
||||
summ = data["summary"]
|
||||
print(
|
||||
f"{'Total / Overall':<25} {summ['total_cases']:>8} {summ['passed_cases']:>8} {summ['failed_cases']:>8} "
|
||||
f"{summ['pass_rate_pct']:>9.1f}% {data['metadata']['total_duration_ms']:>10.2f}ms"
|
||||
)
|
||||
print("=" * 80)
|
||||
|
||||
if self.show_diffs and data["failures"]:
|
||||
print(f"\nDiagnostic Gap Details ({len(data['failures'])} failures pending M2/M3):")
|
||||
for item in data["failures"]:
|
||||
print(f" * [{item['id']}] {item['filename']} ({item['domain']} / {item['edge_case_type']})")
|
||||
for k, v in item["diffs"].items():
|
||||
print(f" - {k}: expected={v['expected']!r}, got={v['actual']!r}")
|
||||
|
||||
print(f"\nSummary JSON exported to: {self.json_output_path.resolve()}\n")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Offline E2E Benchmark Runner for Media Sorter Pattern Recognition."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--domain",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Filter execution by specific domain (e.g. 'Anime', 'Standard TV', 'Movies').",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--case",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Filter execution by specific BenchmarkCase ID (e.g. 'TV-01', 'ANIME-04').",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--json-output",
|
||||
type=str,
|
||||
default=str(DEFAULT_JSON_PATH),
|
||||
help="Output JSON summary destination path.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--verbose",
|
||||
"-v",
|
||||
action="store_true",
|
||||
help="Enable verbose output reporting.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no-diffs",
|
||||
action="store_true",
|
||||
help="Suppress detailed failure diff output.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--strict",
|
||||
action="store_true",
|
||||
help="Exit with non-zero status code if any benchmark case fails.",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
selected_cases = BENCHMARK_CASES
|
||||
if args.domain:
|
||||
selected_cases = [c for c in selected_cases if c.domain.lower() == args.domain.lower()]
|
||||
if not selected_cases:
|
||||
print(f"Error: No benchmark cases match domain '{args.domain}'", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
if args.case:
|
||||
selected_cases = [c for c in selected_cases if c.id.upper() == args.case.upper()]
|
||||
if not selected_cases:
|
||||
print(f"Error: No benchmark case matches ID '{args.case}'", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
runner = BenchmarkRunner(
|
||||
cases=selected_cases,
|
||||
json_output_path=Path(args.json_output),
|
||||
verbose=args.verbose,
|
||||
show_diffs=not args.no_diffs,
|
||||
strict=args.strict,
|
||||
)
|
||||
|
||||
summary_data = runner.run()
|
||||
runner.print_terminal_report(summary_data)
|
||||
|
||||
if args.strict and summary_data["summary"]["failed_cases"] > 0:
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,56 @@
|
||||
"""Automated E2E Benchmark Test Suite (Requirement R2).
|
||||
|
||||
Executes 100% offline with zero network connectivity and zero mutations to real disk storage.
|
||||
Parametrized across all 60+ benchmark cases across the 6 media domains:
|
||||
1. Standard TV
|
||||
2. Anime
|
||||
3. Movies
|
||||
4. Specials & Extras
|
||||
5. Daily / Dated Shows
|
||||
6. Messy & Complex
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
import pytest
|
||||
|
||||
# Ensure project root is in sys.path regardless of how pytest is invoked
|
||||
_PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
|
||||
if str(_PROJECT_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(_PROJECT_ROOT))
|
||||
|
||||
try:
|
||||
from tests.benchmark.benchmark_cases import (
|
||||
BENCHMARK_CASES,
|
||||
BenchmarkCase,
|
||||
evaluate_benchmark_case,
|
||||
)
|
||||
except ImportError:
|
||||
from benchmark_cases import ( # type: ignore
|
||||
BENCHMARK_CASES,
|
||||
BenchmarkCase,
|
||||
evaluate_benchmark_case,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", BENCHMARK_CASES, ids=lambda c: c.id)
|
||||
def test_benchmark(case: BenchmarkCase) -> None:
|
||||
"""Execute end-to-end benchmark test for an individual media file pattern.
|
||||
|
||||
Tests FilenameTokenizer tokenization, MediaClassifier classification, and
|
||||
MediaNamer destination path generation against expected baseline values.
|
||||
"""
|
||||
result = evaluate_benchmark_case(case)
|
||||
|
||||
if not result.passed:
|
||||
error_lines = [
|
||||
f"Benchmark case {case.id} ({case.domain} - {case.edge_case_type}) failed:",
|
||||
f" Filename: {case.filename}",
|
||||
]
|
||||
for field_name, mismatch in result.diffs.items():
|
||||
error_lines.append(
|
||||
f" * {field_name}: expected={mismatch['expected']!r}, got={mismatch['actual']!r}"
|
||||
)
|
||||
pytest.fail("\n".join(error_lines))
|
||||
Reference in new issue
Block a user