Made heuristic faster by iterating each fixing sha once

Data/Database/db_repository.py

@@ -513,6 +513,82 @@ class DBRepository:
         finally:
             self.close()
 
+    def update_heuristic_mapping(
+        self, fixing_sha: str, fixing_config_code: str, heuristic_mapping: str | None, confidence_value: bool
+    ):
+        """
+        Updates the heuristic mapping for a fixing commit.
+        :param fixing_sha: Fixing commit
+        :param fixing_config_code: config code where the fixing commit was found
+        :param heuristic_mapping: Mapping which is the result of the heuristic.
+        :param confidence_value: Confidence value of the mapping
+        """
+        if not self.cnx.is_connected():
+            self._open()
+        cursor = self.cnx.cursor()
+
+        try:
+            update = (
+                "UPDATE `link_fixing_commit_vcc` "
+                "SET `confidence_value` = %(confidence_value)s, `heuristic_mapping` = %(heuristic_mapping)s "
+                "WHERE `fixing_sha` = %(fixing_sha)s AND `fixing_config_code` = %(fixing_config_code)s;"
+            )
+
+            cursor.execute(
+                update,
+                {
+                    "fixing_sha": fixing_sha,
+                    "fixing_config_code": fixing_config_code,
+                    "confidence_value": confidence_value,
+                    "heuristic_mapping": heuristic_mapping,
+                },
+            )
+
+            self.cnx.commit()
+        except mysql.connector.Error as err:
+            warnings.warn(
+                f"Mapping between vcc and fixing commit for {fixing_sha} could not be updated - {str(err)}... Rolling back"
+            )
+            self.cnx.rollback()
+        finally:
+            self.close()
+
+    def update_vcc(self, fixing_sha: str, fixing_config_code, vcc_sha: str):
+        """
+        Updates the VCC of a fixing commit.
+        :param fixing_sha: Fixing commit
+        :param fixing_config_code: config code where the fixing commit was found
+        :param vcc_sha: VCC
+        """
+        if not self.cnx.is_connected():
+            self._open()
+        cursor = self.cnx.cursor()
+
+        try:
+            update = (
+                "UPDATE `link_fixing_commit_vcc` "
+                "SET `vcc_sha` = %(vcc_sha)s "
+                "WHERE `fixing_sha` = %(fixing_sha)s AND `fixing_config_code` = %(fixing_config_code)s;"
+            )
+
+            cursor.execute(
+                update,
+                {
+                    "fixing_sha": fixing_sha,
+                    "fixing_config_code": fixing_config_code,
+                    "vcc_sha": vcc_sha,
+                },
+            )
+
+            self.cnx.commit()
+        except mysql.connector.Error as err:
+            warnings.warn(
+                f"Mapping between vcc and fixing commit for {fixing_sha} could not be updated - {str(err)}... Rolling back"
+            )
+            self.cnx.rollback()
+        finally:
+            self.close()
+
     def get_all_syzkaller_ids(self) -> list:
         """
         :return: a list of all syzkaller ids that are saved in the database
@@ -584,7 +660,10 @@ class DBRepository:
             self._open()
         cursor = self.cnx.cursor(dictionary=True)
 
-        query = "SELECT * FROM `link_fixing_commit_vcc` WHERE `heuristic_mapping` IS NULL;"
+        query = (
+            "SELECT DISTINCT `fixing_sha`, `fixing_config_code`, `cve_id` "
+            "FROM `link_fixing_commit_vcc` WHERE `heuristic_mapping` IS NULL;"
+        )
 
         cursor.execute(query)
         result = cursor.fetchall()

Data/Heuristic/heuristic_parallel.py

@@ -37,24 +37,20 @@ def run_heuristic(mapping_chunk):
             )
             confidence = calculate_confidence(results, cve)
             result_json = {c.hexsha: i for c, i in results.items()}
-            db_repo.update_vcc_fixing_commit(
-                mapping["mapping_id"],
-                mapping["cve_id"],
-                mapping["vcc_sha"],
+            db_repo.update_heuristic_mapping(
+                mapping["fixing_sha"],
                 mapping["fixing_config_code"],
-                confidence,
                 json.dumps(result_json),
+                confidence
             )
         else:
             print(f"PID {os.getpid()} ({mapping['fixing_config_code']}): No VCCs found for {mapping['fixing_sha']}")
             # If no VCCs are found, we still want to update the mapping to indicate that we have checked it
-            db_repo.update_vcc_fixing_commit(
-                mapping["mapping_id"],
-                mapping["cve_id"],
-                mapping["vcc_sha"],
+            db_repo.update_heuristic_mapping(
+                mapping["fixing_sha"],
                 mapping["fixing_config_code"],
-                False,
-                json.dumps({})
+                json.dumps({}),
+                False
             )
     print(f"PID {os.getpid()} Finished heuristic range")
 

Data/Heuristic/heuristic_sequential.py

@@ -38,23 +38,19 @@ def main():
             tqdm.write(f"Found {len(results)} possible VCCs for {mapping['fixing_sha']}")
             confidence = calculate_confidence(results, cve)
             result_json = {c.hexsha: i for c, i in results.items()}
-            db_repo.update_vcc_fixing_commit(
-                mapping["mapping_id"],
-                mapping["cve_id"],
-                mapping["vcc_sha"],
+            db_repo.update_heuristic_mapping(
+                mapping["fixing_sha"],
                 mapping["fixing_config_code"],
-                confidence,
                 json.dumps(result_json),
+                confidence
             )
         else:
             # If no VCCs are found, we still want to update the mapping to indicate that we have checked it
-            db_repo.update_vcc_fixing_commit(
-                mapping["mapping_id"],
-                mapping["cve_id"],
-                mapping["vcc_sha"],
+            db_repo.update_heuristic_mapping(
+                mapping["fixing_sha"],
                 mapping["fixing_config_code"],
-                False,
-                json.dumps({})
+                json.dumps({}),
+                False
             )