commit 666f93fac2cece64606f66d1b457ae0a009cd59a
parent 1444dbed1c9127352025ed4ab0799ba41c7da2d7
Author: rexxnor <rexxnor+gnunet@brief.li>
Date: Fri, 12 Oct 2018 11:42:14 +0200
fixed serial fetching and added serialization of zone
Diffstat:
1 file changed, 60 insertions(+), 31 deletions(-)
diff --git a/gnsmigrator/gnsmigrator.py b/gnsmigrator/gnsmigrator.py
@@ -23,6 +23,8 @@ from ctypes import c_void_p
from enum import Enum
from dataclasses import dataclass
import multiprocessing
+import gzip
+import pickle
import queue
import re
import sys
@@ -395,12 +397,15 @@ class BaseMigrator():
"""
Extracts the current serial from a given zone
"""
+ if zonename[-1] == '.':
+ zonename = zonename[:-1]
serial = sp.check_output([GNUNET_GNS_COMMAND,
'-t', 'SOA',
'-u', '@.%s' % zonename])
- soapattern = re.compile(r'.+\s(\d+,)\d+,+\d+,\d+,\d+')
- if re.match(soapattern, serial):
- soa_serial = re.match(soapattern, serial)
+ serial = serial.decode()
+ soapattern = re.compile(r'.+\s(\d+),\d+,+\d+,\d+,\d+', re.M)
+ if re.findall(soapattern, serial):
+ soa_serial = re.findall(soapattern, serial)[0]
else:
soa_serial = 0
return soa_serial
@@ -427,11 +432,17 @@ class TLDMigrator(BaseMigrator):
cls.zonegenerator = {}
@classmethod
- def initial_zone_transfer(cls):
+ def initial_zone_transfer(cls, serial):
"""
Transfer and initialize the zone
"""
- cls.zonegenerator = dns.query.xfr(cls.transferns, cls.tld)
+ if serial:
+ cls.zonegenerator = dns.query.xfr(cls.transferns,
+ cls.tld,
+ rdtype=dns.rdatatype.IXFR,
+ serial=serial)
+ else:
+ cls.zonegenerator = dns.query.xfr(cls.transferns, cls.tld,)
@classmethod
def bootstrap_zone(cls):
@@ -455,33 +466,49 @@ class TLDMigrator(BaseMigrator):
Extract necessary information from Generator
"""
zone = None
+ # And to unzip and unpickle:
try:
- for message in cls.zonegenerator:
- origin = message.origin
- rdclass = message.answer[0].rdclass
- if zone is None:
- if relativize:
- origin = message.origin
- else:
- origin = message.answer[0].name
+ with gzip.open('%s.zone.gz' % cls.tld[:-1], 'rb') as zonefile:
+ zone = pickle.load(zonefile)
+ except Exception:
+ pass
+
+ currentserial = cls.get_current_serial(cls.tld)
+ zoneserial = None
+ if zone:
+ zoneserial = zone[0][2].split(' ')[2].split(',')[0]
+ if currentserial != zoneserial:
+ try:
+ for message in cls.zonegenerator:
+ origin = message.origin
rdclass = message.answer[0].rdclass
- zone = zone_factory(origin, rdclass, relativize=relativize)
- for rrset in message.answer:
- znode = zone.nodes.get(rrset.name)
- if not znode:
- znode = zone.node_factory()
- zone.nodes[rrset.name] = znode
- zrds = znode.find_rdataset(rrset.rdclass, rrset.rdtype,
- rrset.covers, True)
- zrds.update_ttl(rrset.ttl)
- for record in rrset:
- record.choose_relativity(zone.origin, relativize)
- zrds.add(record)
- if check_origin:
- zone.check_origin()
- cls.zone = zone
- except Exception as transferexception:
- print("Error occured during Zone transfer: %s" % transferexception)
+ if zone is None:
+ if relativize:
+ origin = message.origin
+ else:
+ origin = message.answer[0].name
+ rdclass = message.answer[0].rdclass
+ zone = zone_factory(origin, rdclass, relativize=relativize)
+ for rrset in message.answer:
+ znode = zone.nodes.get(rrset.name)
+ if not znode:
+ znode = zone.node_factory()
+ zone.nodes[rrset.name] = znode
+ zrds = znode.find_rdataset(rrset.rdclass, rrset.rdtype,
+ rrset.covers, True)
+ zrds.update_ttl(rrset.ttl)
+ for record in rrset:
+ record.choose_relativity(zone.origin, relativize)
+ zrds.add(record)
+ if check_origin:
+ zone.check_origin()
+ cls.zone = zone
+ except Exception as transferexception:
+ print("Error occured during Zone transfer: %s" % transferexception)
+
+ # To zip and pickle:
+ with gzip.open('%s.zone.gz' % cls.tld[:-1], 'wb') as zonefile:
+ pickle.dump(zone, zonefile)
@classmethod
def multithreaded_add_records_to_gns(cls):
@@ -539,6 +566,7 @@ class TLDMigrator(BaseMigrator):
for thread in threads:
thread.join()
+
def main():
"""
Initializes object and handles arguments
@@ -560,7 +588,8 @@ def main():
if tld and transferns:
migrator = TLDMigrator(tld, transferns)
- migrator.initial_zone_transfer()
+ serial = migrator.get_current_serial(tld)
+ migrator.initial_zone_transfer(serial)
migrator.bootstrap_zone()
migrator.mirror_zone()
migrator.multithreaded_add_records_to_gns()
