commit 900f93c431db4ef2455e159d9c98261083407196
parent 2829a7614d724fb1e2f01f81d9afd0661dd2e553
Author: rexxnor <rexxnor+gnunet@brief.li>
Date: Mon, 8 Oct 2018 12:46:54 +0200
created baseclass and separated small from big zones
Diffstat:
1 file changed, 183 insertions(+), 74 deletions(-)
diff --git a/gnsmigrator/gnsmigrator.py b/gnsmigrator/gnsmigrator.py
@@ -2,24 +2,25 @@
"""GNS Migrator
Usage:
- gnsmigrator.py (-c <csv> | -f <txtfile>)
- gnsmigrator.py (-c <csv> | -f <txtfile>) -r <resolver>
+ gnsmigrator.py -t <tld> -ns <transferns>
+ gnsmigrator.py -f <txtfile>
gnsmigrator.py -h | --help
gnsmigrator.py -v | --version
Options:
- <csv> CSV File containing domains to transfer
+ <tld> Top level domain to migrate
<txtfile> Text File containing domains to transfer
- <resolver> DNS Server that resolves missing domains
+ <transferns> DNS Server that does the zone transfer
-h --help Show this screen.
-v --version Show version.
"""
# imports
+import multiprocessing
+import queue
import sys
-import time
import subprocess
-import csv
+import threading
import dns.query
import dns.resolver
import dns.zone
@@ -31,9 +32,9 @@ GNUNET_NAMESTORE_COMMAND = 'gnunet-namestore'
GNUNET_GNS_COMMAND = 'gnunet-gns'
GNUNET_ARM_COMMAND = 'gnunet-arm'
-class GNSMigrator():
+class BaseMigrator():
"""
- Class that provides functionality to migrate zones
+ Base class for migration
"""
@classmethod
def __init__(cls, domainlist):
@@ -65,7 +66,46 @@ class GNSMigrator():
continue
cls.zones[domain] = (zone, (master_answer[0].address,
domain,
- zone.get_rdataset('@', dns.rdatatype.SOA).ttl))
+ zone.get_rdataset('@', dns.rdatatype.SOA).ttl,
+ 0))
+
+ @classmethod
+ def refresh_zone(cls, domain, zonetuple, dnsresolver):
+ """
+ Refresh the zone using IXFR and the previous serial as reference
+
+ :param domain: The domain to transfer and migrate
+ :param zonetuple: The necessary data tuple for the transfer
+ :param dnsresolver: Optional user specified resolver for subdomains
+ """
+ zone, xfrinfo = zonetuple
+ zonename = cls.get_lowest_domain_part(domain)
+ cls.add_records_to_gns(zonename, zone, domain, dnsresolver)
+ newzone = dns.zone.Zone(domain)
+
+ # Ugly way to get serial
+ if xfrinfo[3] == 0:
+ oldserial = 0
+ else:
+ oldserial = int(str(zone.get_rdataset('@', dns.rdatatype.SOA)).split(' ')[5])
+ xfrinfo[3] = 1
+
+ # A normal BIND9 returns a normal AXFR response with the entire zone
+ # if the serial is newer. This is why there is no real incremental
+ # zone transfer using bind. This makes the merger_zones function
+ # unnecessary. Furthermore this try except block updates only if
+ # there is a newer zone availible (according to serial). The IXFR
+ # returns only a SOA record with a new serial if it has not changed
+ try:
+ newzone = dns.zone.from_xfr(dns.query.xfr(xfrinfo[0],
+ xfrinfo[1],
+ rdtype=dns.rdatatype.IXFR,
+ serial=oldserial))
+ cls.zones[domain] = (newzone, (xfrinfo[0],
+ xfrinfo[1],
+ zone.get_rdataset('@', dns.rdatatype.SOA).ttl))
+ except dns.zone.NoNS:
+ print('the zone for domain %s was not updated' % domain)
@classmethod
def bootstrap_zones(cls):
@@ -76,6 +116,7 @@ class GNSMigrator():
counter = 0
# building list with arguments
reverse_parsing = domain.split('.')[::-1]
+ reverse_parsing = list(filter(None, reverse_parsing))
for domainpart in reverse_parsing:
pkey_lookup = subprocess.Popen([GNUNET_ZONE_CREATION_COMMAND,
'-d'],
@@ -190,7 +231,6 @@ class GNSMigrator():
'-V', '%s.%s' % (dnsname_str, domain),
'-e', '%ds' % ttl])
-
@staticmethod
def get_lowest_domain_part(domain):
"""
@@ -198,62 +238,136 @@ class GNSMigrator():
"""
return domain.split('.')[0]
- @staticmethod
- def merge_zones(domain, fullzone, incrementalzone):
- """
- Merges full zone with incremental zone
- """
- # The library sucks so I do it with string operations
- fullset = set(fullzone.to_text().decode().split('\n'))
- incrementalset = set(incrementalzone.to_text().decode().split('\n'))
- merged = '\n'.join(fullset.union(incrementalset))
- mergedzone = dns.zone.from_text(merged.encode(), origin=domain)
- return mergedzone
+class ZoneMigrator(BaseMigrator):
+ """
+ Class that migrates small zones efficiently
+ """
+ @classmethod
+ def __init__(cls, domainlist):
+ BaseMigrator.__init__(domainlist)
+class TLDMigrator(BaseMigrator):
+ """
+ Class that migrates big zones (TLDs) efficiently
+ """
+ @classmethod
+ def __init__(cls, tld, transferns):
+ BaseMigrator.__init__(tld)
+ cls.tld = tld
+ cls.transferns = transferns
+ cls.zone = None
+ cls.zonegenerator = None
@classmethod
- def refresh_zone(cls, domain, zonetuple, dnsresolver):
+ def initial_zone_transfer(cls):
"""
- Refresh the zone using IXFR and the previous serial as reference
-
- :param domain: The domain to transfer and migrate
- :param zonetuple: The necessary data tuple for the transfer
- :param dnsresolver: Optional user specified resolver for subdomains
+ Transfer and initialize the zone
"""
- zone, xfrinfo = zonetuple
- zonename = cls.get_lowest_domain_part(domain)
- cls.add_records_to_gns(zonename, zone, domain, dnsresolver)
- newzone = dns.zone.Zone(domain)
+ cls.zonegenerator = dns.query.xfr(cls.transferns, cls.tld)
- # Ugly way to get serial
- oldserial = int(str(zone.get_rdataset('@', dns.rdatatype.SOA)).split(' ')[5])
+ @classmethod
+ def bootstrap_zone(cls):
+ """
+ Creates the zone in gnunet
+ """
+ reverse_parsing = cls.tld.split('.')[::-1]
+ reverse_parsing = list(filter(None, reverse_parsing))
+ for domainpart in reverse_parsing:
+ try:
+ subprocess.run([GNUNET_ZONE_CREATION_COMMAND,
+ '-C', domainpart])
+ except subprocess.CalledProcessError:
+ print("Zone %s already exists!" % domainpart)
- # A normal BIND9 returns a normal AXFR response with the entire zone
- # if the serial is newer. This is why there is no real incremental
- # zone transfer using bind. This makes the merger_zones function
- # unnecessary. Furthermore this try except block updates only if
- # there is a newer zone availible (according to serial). The IXFR
- # returns only a SOA record with a new serial if it has not changed
+ @classmethod
+ def mirror_zone(cls, zone_factory=dns.zone.Zone, relativize=True, check_origin=True):
+ """
+ Extract necessary information from Generator
+ """
+ zone = None
try:
- newzone = dns.zone.from_xfr(dns.query.xfr(xfrinfo[0],
- xfrinfo[1],
- rdtype=dns.rdatatype.IXFR,
- serial=oldserial))
- cls.zones[domain] = (newzone, (xfrinfo[0],
- xfrinfo[1],
- zone.get_rdataset('@', dns.rdatatype.SOA).ttl))
- except dns.zone.NoNS:
- print('the zone for domain %s was not updated' % domain)
+ for message in cls.zonegenerator:
+ origin = message.origin
+ rdclass = message.answer[0].rdclass
+ if zone is None:
+ if relativize:
+ origin = message.origin
+ else:
+ origin = message.answer[0].name
+ rdclass = message.answer[0].rdclass
+ zone = zone_factory(origin, rdclass, relativize=relativize)
+ for rrset in message.answer:
+ znode = zone.nodes.get(rrset.name)
+ if not znode:
+ znode = zone.node_factory()
+ zone.nodes[rrset.name] = znode
+ zrds = znode.find_rdataset(rrset.rdclass, rrset.rdtype,
+ rrset.covers, True)
+ zrds.update_ttl(rrset.ttl)
+ for record in rrset:
+ record.choose_relativity(zone.origin, relativize)
+ zrds.add(record)
+ if check_origin:
+ zone.check_origin()
+ cls.zone = zone
+ except Exception as e:
+ print("Error occured during Zone transfer: %s" % e)
- # Merge old and new zone
- # updatedzone = cls.merge_zones(domain, zone, newzone)
+ @classmethod
+ def multithreaded_add_records_to_gns(cls):
+ """
+ Extracts records from zone and adds them to GNS
+ """
+ print("Starting to add records into GNS...")
+ zonename = cls.tld.split('.')[0]
+
+ # Defining FIFO Queue
+ taskqueue = queue.Queue(maxsize=200)
+ # Defining worker
+ def worker():
+ while True:
+ record = taskqueue.get()
+ if record is None:
+ break
+ # execute thing to run on item
+ dnsname, ttl, authns = record
+ authns = str(authns)[:-1]
+ subprocess.run([GNUNET_NAMESTORE_COMMAND,
+ '-z', zonename,
+ '-a', '-n', str(dnsname),
+ '-t', 'GNS2DNS',
+ '-V', '%s.%s@%s' % (str(dnsname),
+ zonename,
+ str(authns)),
+ '-e', '%ds' % int(ttl)])
+ taskqueue.task_done()
+
+ # Create threads
+ threads = []
+ for _ in range(multiprocessing.cpu_count()):
+ thread = threading.Thread(target=worker)
+ thread.start()
+ threads.append(thread)
+
+ # Give workers stuff to do
+ for record in cls.zone.iterate_rdatas(rdtype=dns.rdatatype.NS):
+ taskqueue.put(record)
+
+ # Block until all tasks are done
+ taskqueue.join()
+
+ # Stop workers
+ for _ in range(multiprocessing.cpu_count()):
+ taskqueue.put(None)
+ for thread in threads:
+ thread.join()
def main():
"""
Initializes object and handles arguments
"""
# argument parsing from docstring definition
- args = docopt.docopt(__doc__, version='GNS Migrator 0.1.1')
+ args = docopt.docopt(__doc__, version='GNS Migrator 0.1.3')
# Checks if GNUnet services are running
try:
@@ -261,35 +375,30 @@ def main():
except subprocess.TimeoutExpired:
print('GNUnet Services are not running!')
print('Exiting...')
- return 1
+ sys.exit(1)
dnsresolver = args.get('<resolver>', None)
-
- domainlist = []
-
- if args.get('<csv>', None):
- csvfile = args['<csv>']
- with open(csvfile, 'r') as openedcsv:
- linereader = csv.reader(openedcsv, delimiter=' ', quotechar='|')
- for domain in linereader:
- domainlist += domain
-
- if args.get('<txtfile>', None):
+ tld = args.get('<tld>', None)
+ transferns = args.get('<transferns>', None)
+ txtfile = args.get('<txtfile>', None)
+
+ if tld and transferns:
+ migrator = TLDMigrator(tld, transferns)
+ migrator.initial_zone_transfer()
+ migrator.bootstrap_zone()
+ migrator.mirror_zone()
+ migrator.multithreaded_add_records_to_gns()
+ elif txtfile:
+ domainlist = []
txtfile = args['<txtfile>']
with open(txtfile, 'r') as openedtxt:
for line in openedtxt:
domainlist.append(line.rstrip())
-
-
- gnsmigrator = GNSMigrator(domainlist)
- gnsmigrator.bootstrap_zones()
- gnsmigrator.initial_zone_transfer()
-
-
- # TODO add a daemon for doing this and notify when the TTL has expired
- for domain, zonetuple in gnsmigrator.zones.items():
- # Returns a value 0 if not changed and 1 if changed
- gnsmigrator.refresh_zone(domain, zonetuple, dnsresolver)
+ zonemigrator = ZoneMigrator(domainlist)
+ zonemigrator.initial_zone_transfer()
+ zonemigrator.bootstrap_zones()
+ for domain, zonetuple in zonemigrator.zones.items():
+ zonemigrator.refresh_zone(domain, zonetuple, dnsresolver)
if __name__ == '__main__':
main()
