data_lib.py

@@ -68,7 +68,7 @@ class Run:
         return self.algo_version == self.algo.current_version
 
 class Data:
-    def __init__(self, logfile=None, validate_versions=True):
+    def __init__(self, logfile=None, validate_versions=True, remove_duplications=False):
         self.validate_versions = validate_versions
         logfile = logfile or os.environ.get('LOGFILE', 'log')
         raw_json = "[" + open(logfile,mode='r').read()[0:-2] + "]"
@@ -78,10 +78,17 @@ class Data:
         self.pipelines = {}
         self.runs = []
 
+        duplication_checker = {}
+
         for it in raw_data:
             run = Run(self, it)
             self.runs.append(run)
             if not validate_versions or run.is_up_to_date():
+                if remove_duplications:
+                    key = (run.n, run.seed, run.print_pipeline())
+                    if key in duplication_checker:
+                        continue
+                    duplication_checker[key] = run
                 self.pipelines.setdefault(run.print_pipeline(), [])
                 self.pipelines[run.print_pipeline()].append(run)
 

prace/bakalarka/g.py

@@ -9,7 +9,7 @@ import sys, os
 d = pathlib.Path("/".join(__file__.split("/")[:-1]))
 
 def load(name):
-    return data_lib.Data(d/name, validate_versions=False)
+    return data_lib.Data(d/name, validate_versions=False, remove_duplications=True)
 
 def load_main_test():
     return load(d/"main_test/log")