ascension

commit 1990ea7518d7ef0de3a8743ede70f7d3a3efdb0a
parent 7623c3bd29f73afc7aaac07192e2f497736f5602
Author: rexxnor <rexxnor+gnunet@brief.li>
Date:   Sun, 30 Jun 2019 20:31:09 +0200

added some performance metrics and prevented transferring zone twice

Diffstat:
M
ascension/ascension.py
 | 
30
++++++++++++++++++++++++------

1 file changed, 24 insertions(+), 6 deletions(-)
diff --git a/ascension/ascension.py b/ascension/ascension.py
@@ -180,6 +180,9 @@ class Ascender():
                           domain)
             return None
 
+        # This prevents transferring the zone a second time
+        self.zone = zone
+
         for soa_record in zone.iterate_rdatas(rdtype=dns.rdatatype.SOA):
             if not self.transferns:
                 mname = soa_record[2].mname
@@ -197,7 +200,7 @@ class Ascender():
         logging.info("Starting to add records into GNS...")
 
         # Defining FIFO Queue
-        taskqueue = queue.Queue(maxsize=5)
+        taskqueue = queue.Queue(maxsize=10)
 
         # Defining worker
         def worker():
@@ -300,19 +303,27 @@ class Ascender():
                 taskqueue.task_done()
         # End of worker
 
+        #import cProfile
+        start = time.time()
+        #cProfile.runctx('self.create_zone_hierarchy()', globals(), locals())
         self.create_zone_hierarchy()
+        end = time.time()
+        print("Zone hierarchy in %s seconds" % str(end - start))
 
         # Create one thread
         thread = threading.Thread(target=worker)
         thread.start()
 
         # add records
+        start = time.time()
         for name, rdatasets in self.zone.nodes.items():
             # log if the rdataset is empty for some reason
             if not rdatasets:
                 logging.warning("Empty Rdataset!")
                 continue
             taskqueue.put((name, rdatasets))
+        end = time.time()
+        print("Records added in %s seconds" % str(end - start))
 
         # Block until all tasks are done
         taskqueue.join()
@@ -698,11 +709,11 @@ class Ascender():
         # Create missing zones (and add to dict) for GNS zones that are NOT DNS
         # zones ("." is not a zone-cut in DNS, but always in GNS). Only add the
         # records that there are no NS records for
-        remaining_nsrecords = list(filter(lambda name: not name.is_absolute(),
+        remaining_nsrecords = set(filter(lambda name: not name.is_absolute(),
                                           nameserverlist))
-        remaining = list(filter(lambda name: name not in remaining_nsrecords,
+        remaining = set(filter(lambda name: name not in remaining_nsrecords,
                                 self.zone.nodes.keys()))
-        final = list(filter(lambda name: len(str(name).split('.')) > 1,
+        final = set(filter(lambda name: len(str(name).split('.')) > 1,
                             remaining))
 
         for name in final:
@@ -772,6 +783,9 @@ def main():
     retry = 300
     refresh = 300
 
+    # variable to keep state
+    needsupdate = False
+
     # Main loop for actual daemon
     while True:
         gns_zone_serial = ascender.get_gns_zone_serial()
@@ -814,15 +828,18 @@ def main():
             continue
         else:
             logging.info("GNS zone is out of date, performing incremental transfer.")
+            needsupdate = True
             if standalone:
                 return 1
             print("GNS zone is out of date, performing incremental transfer.")
 
         try:
             start = time.time()
-            ascender.zone = dns.zone.from_xfr(ascender.zonegenerator,
-                                              check_origin=False)
+            if not ascender.zone or needsupdate:
+                ascender.zone = dns.zone.from_xfr(ascender.zonegenerator,
+                                                  check_origin=False)
             end = time.time()
+            needsupdate = False
             print("Zone transferrred in %s seconds" % str(end - start))
             ascender.soa = ascender.get_zone_soa(ascender.zone)
             refresh = int(str(ascender.soa[2]).split(" ")[3])
@@ -835,6 +852,7 @@ def main():
             continue
 
         ascender.add_records_to_gns()
+
         logging.info("Added %d RRSets", ascender.rrsetcount)
         logging.info("Finished migration of the zone %s", ascender.domain)