commit 1ff17a356bfb2aeae3eb4dfa45b0c2400be962a6
parent 65d94cf4300305b946f8ab5b78d8a082c6deb3a4
Author: rexxnor <rexxnor+gnunet@brief.li>
Date: Thu, 25 Oct 2018 13:45:04 +0200
refactored code, added debug mode
Diffstat:
1 file changed, 145 insertions(+), 324 deletions(-)
diff --git a/gnsmigrator/gnsmigrator.py b/gnsmigrator/gnsmigrator.py
@@ -2,19 +2,18 @@
"""GNS Migrator
Usage:
- gnsmigrator.py -t <tld> -ns <transferns>
- gnsmigrator.py -t <tld> -ns <transferns> -p <port>
- gnsmigrator.py -f <txtfile>
- gnsmigrator.py -f <txtfile> -p <port>
- gnsmigrator.py -f <txtfile> -p <port> -ns <transferns>
+ gnsmigrator.py <domain> [-d]
+ gnsmigrator.py <domain> -p <port> [-d]
+ gnsmigrator.py <domain> -ns <transferns> [-d]
+ gnsmigrator.py <domain> -ns <transferns> -p <port> [-d]
gnsmigrator.py -h | --help
gnsmigrator.py -v | --version
Options:
- <port> Port specification
- <tld> Top level domain to migrate
- <txtfile> Text File containing domains to transfer
+ <port> Port for zone transfer
+ <domain> Domain to migrate
<transferns> DNS Server that does the zone transfer
+ -d --debug Enable debugging
-h --help Show this screen.
-v --version Show version.
"""
@@ -26,7 +25,6 @@ from ctypes import c_uint64
from ctypes import c_void_p
from enum import Enum
from dataclasses import dataclass
-import multiprocessing
import queue
import re
import sys
@@ -36,6 +34,7 @@ import dns.query
import dns.resolver
import dns.zone
import docopt
+import logging
# GLOBALS
GNUNET_ZONE_CREATION_COMMAND = 'gnunet-identity'
@@ -63,189 +62,100 @@ class GNUnetGNSRecordData():
record_type: c_uint32
flags: GNUnetGNSRecordFlags
-class BaseMigrator():
+class GNSMigrator():
"""
- Base class for migration
+ Class that provides migration for any given domain
"""
@classmethod
- def __init__(cls, domainlist, port=53):
- cls.domainlist = domainlist
+ def __init__(cls, domain, transferns, port):
+ cls.domain = domain
cls.port = port
- cls.zones = {}
-
- @classmethod
- def initial_zone_transfer(cls, resolver=None):
- """
- Fetch all the zones via zone transfer
- """
- for domain in cls.domainlist:
- try:
- soa_answer = dns.resolver.query(domain, 'SOA')
- except dns.resolver.NoAnswer:
- print("the domain '%s' does not exist" % domain)
- continue
-
- master_answer = dns.resolver.query(soa_answer[0].mname, 'A')
-
- try:
- if resolver:
- zone = dns.zone.from_xfr(dns.query.xfr(
- resolver, domain, port=cls.port))
- else:
- zone = dns.zone.from_xfr(dns.query.xfr(
- master_answer[0].address, domain,
- port=cls.port))
- except dns.resolver.NoAnswer:
- print("nameserver for '%s' did not answer" % domain)
- continue
- except dns.exception.FormError:
- print("domain '%s' does not allow xfr requests" % domain)
- continue
- if resolver:
- cls.zones[domain] = (zone,
- (resolver,
- domain,
- zone.get_rdataset('@',
- dns.rdatatype.SOA).ttl,
- 0)
- )
- else:
- cls.zones[domain] = (zone,
- (master_answer[0].address,
- domain,
- zone.get_rdataset('@',
- dns.rdatatype.SOA).ttl,
- 0)
- )
+ cls.soa = None
+ cls.transferns = transferns
+ cls.zone = None
+ cls.zonegenerator = None
@classmethod
- def refresh_zone(cls, domain, zonetuple):
+ def initial_zone_transfer(cls, serial=None):
"""
- Refresh the zone using IXFR and the previous serial as reference
-
- :param domain: The domain to transfer and migrate
- :param zonetuple: The necessary data tuple for the transfer
- :param dnsresolver: Optional user specified resolver for subdomains
+ Transfer and initialize the zone
"""
- zone, xfrinfo = zonetuple
- zonename = cls.get_lowest_domain_part(domain)
- cls.add_records_to_gns(zonename, zone, domain)
- newzone = dns.zone.Zone(domain)
-
- # Ugly way to get serial
- if xfrinfo[3] == 0:
- oldserial = 0
+ if serial:
+ cls.zonegenerator = dns.query.xfr(cls.transferns,
+ cls.domain,
+ rdtype=dns.rdatatype.IXFR,
+ serial=serial,
+ port=cls.port)
else:
- oldserial = int(str(zone.get_rdataset('@', dns.rdatatype.SOA))
- .split(' ')[5])
- xfrinfo[3] = 1
+ cls.zonegenerator = dns.query.xfr(cls.transferns,
+ cls.domain,
+ port=cls.port)
- try:
- newzone = dns.zone.from_xfr(dns.query.xfr(xfrinfo[0],
- xfrinfo[1],
- rdtype=dns.rdatatype.IXFR,
- serial=oldserial))
- cls.zones[domain] = (newzone,
- (xfrinfo[0],
- xfrinfo[1],
- zone.get_rdataset('@',
- dns.rdatatype.SOA).ttl))
- except dns.zone.NoNS:
- print('the zone for domain %s was not updated' % domain)
@classmethod
- def bootstrap_zones(cls):
- """
- Bootstrap the zone structure into GNS
- """
- for domain in cls.domainlist:
- counter = 0
- # building list with arguments
- reverse_parsing = domain.split('.')[::-1]
- reverse_parsing = list(filter(None, reverse_parsing))
- for domainpart in reverse_parsing:
- pkey_lookup = sp.Popen([GNUNET_ZONE_CREATION_COMMAND,
- '-d'],
- stdout=sp.PIPE)
- pkey_line = sp.Popen(['grep', domainpart],
- stdin=pkey_lookup.stdout,
- stdout=sp.PIPE)
- pkey_zone = sp.check_output(['cut', '-d',
- ' ', '-f3'],
- stdin=pkey_line.stdout)
- pkey_zone = pkey_zone.decode().strip()
- pkey_lookup.stdout.close()
- pkey_line.stdout.close()
- # Create identity in GNUnet
- if not pkey_zone:
- sp.run([GNUNET_ZONE_CREATION_COMMAND,
- '-C', domainpart])
-
- pkey_lookup = sp.Popen([GNUNET_ZONE_CREATION_COMMAND,
- '-d'],
- stdout=sp.PIPE)
- pkey_line = sp.Popen(['grep', domainpart],
- stdin=pkey_lookup.stdout,
- stdout=sp.PIPE)
- pkey_zone = sp.check_output(['cut', '-d', ' ', '-f3'],
- stdin=pkey_line.stdout)
- pkey_zone = pkey_zone.decode().strip()
- pkey_lookup.stdout.close()
- pkey_line.stdout.close()
-
- # If it is TLD, don't add PKEY to higher zone
- if counter > 0:
- result = sp.check_output([GNUNET_GNS_COMMAND,
- '-t', 'PKEY',
- '-u', '%s.%s' %
- (domainpart,
- reverse_parsing[counter - 1])])
-
- if "No results." in result.decode():
- sp.run([GNUNET_NAMESTORE_COMMAND,
- '-z', reverse_parsing[counter - 1],
- '-a', '-n', domainpart,
- '-t', 'PKEY',
- '-V', pkey_zone,
- '-e', 'never'])
- counter += 1
-
- @staticmethod
- def add_records_to_gns(zonename, zone, domain):
+ def bootstrap_zone(cls):
"""
- Checks if records are present and adds them if not
- :param zonename: zonename of zone to add records to
- :param zone: the transfered zone
- :param domain: full domain of zone
+ Creates the zone in gnunet
"""
- for record in zone.iterate_rdatas():
- dnsname_str = str(record[0])
- rtype_str = dns.rdatatype.to_text(record[2].rdtype)
- if dnsname_str == '@':
- if rtype_str == 'SOA':
- BaseMigrator.add_soa_record_to_gns(record, zonename, domain)
- else:
- if rtype_str == 'NS' and dnsname_str != '@':
- BaseMigrator.add_ns_record_to_gns(record, zonename, domain)
- elif rtype_str == 'MX':
- BaseMigrator.add_mx_record_to_gns(record, zonename)
- # TODO Add support for SRV records
- #elif rtype_str == 'SRV':
- # BaseMigrator.add_srv_record_to_gns(record, zonename)
- elif rtype_str in ['A', 'AAAA']:
- BaseMigrator.add_a_aaaa_record_to_gns(record, zonename, domain)
- elif rtype_str in ['TXT', 'CNAME']:
- BaseMigrator.add_gen_record_to_gns(record, zonename)
- else:
- print("Record type %s is not yet supported" % rtype_str)
-
+ reverse_parsing = cls.domain.split('.')[::-1]
+ reverse_parsing = list(filter(None, reverse_parsing))
+ counter = 0
+ for domainpart in reverse_parsing:
+ pkey_lookup = sp.Popen([GNUNET_ZONE_CREATION_COMMAND,
+ '-d'],
+ stdout=sp.PIPE)
+ pkey_line = sp.Popen(['grep', domainpart],
+ stdin=pkey_lookup.stdout,
+ stdout=sp.PIPE)
+ pkey_zone = sp.check_output(['cut', '-d',
+ ' ', '-f3'],
+ stdin=pkey_line.stdout)
+ pkey_zone = pkey_zone.decode().strip()
+ pkey_lookup.stdout.close()
+ pkey_line.stdout.close()
+ # Create identity in GNUnet
+ try:
+ ret = sp.run([GNUNET_ZONE_CREATION_COMMAND,
+ '-C', domainpart],
+ stdout=sp.DEVNULL,
+ stderr=sp.DEVNULL)
+ except sp.CalledProcessError:
+ logging.info("Zone %s already exists!" % domainpart)
+ pkey_lookup = sp.Popen([GNUNET_ZONE_CREATION_COMMAND,
+ '-d'],
+ stdout=sp.PIPE)
+ pkey_line = sp.Popen(['grep', domainpart],
+ stdin=pkey_lookup.stdout,
+ stdout=sp.PIPE)
+ pkey_zone = sp.check_output(['cut', '-d', ' ', '-f3'],
+ stdin=pkey_line.stdout)
+ pkey_zone = pkey_zone.decode().strip()
+ pkey_lookup.stdout.close()
+ pkey_line.stdout.close()
+
+ # If it is TLD, don't add PKEY to higher zone
+ if counter > 0:
+ result = sp.check_output([GNUNET_GNS_COMMAND,
+ '-t', 'PKEY',
+ '-u', '%s.%s' %
+ (domainpart,
+ reverse_parsing[counter - 1])])
+
+ if "No results." in result.decode():
+ sp.run([GNUNET_NAMESTORE_COMMAND,
+ '-z', reverse_parsing[counter - 1],
+ '-a', '-n', domainpart,
+ '-t', 'PKEY',
+ '-V', pkey_zone,
+ '-e', 'never'])
+ counter += 1
@staticmethod
def add_gen_record_to_gns(record, zonename):
"""
Adds a generic record to GNS
"""
- if not BaseMigrator.check_if_record_exists_in_zone(record, zonename):
+ if not GNSMigrator.check_if_record_exists_in_zone(record, zonename):
dnsname, ttl, rdata = record
rtype_str = str(dns.rdatatype.to_text(rdata.rdtype))
sp.run([GNUNET_NAMESTORE_COMMAND,
@@ -261,7 +171,7 @@ class BaseMigrator():
"""
Adds a SRV record to GNS
"""
- if not BaseMigrator.check_if_record_exists_in_zone(record, zonename):
+ if not GNSMigrator.check_if_record_exists_in_zone(record, zonename):
value, ttl, rdata = record
rtype_str = str(dns.rdatatype.to_text(rdata.rdtype))
dnsname_str = str(rdata).split(' ')[3]
@@ -284,7 +194,7 @@ class BaseMigrator():
"""
Adds A and AAAA records to GNS
"""
- if not BaseMigrator.check_if_record_exists_in_zone(record, zonename):
+ if not GNSMigrator.check_if_record_exists_in_zone(record, zonename):
dnsname, ttl, rdata = record
rtype_str = str(dns.rdatatype.to_text(rdata.rdtype))
sp.run([GNUNET_NAMESTORE_COMMAND,
@@ -330,7 +240,7 @@ class BaseMigrator():
"""
Adds a GNS2DNS record to GNS
"""
- if not BaseMigrator.check_if_record_exists_in_zone(record, zonename):
+ if not GNSMigrator.check_if_record_exists_in_zone(record, zonename):
dnsname, ttl, rdata = record
nameserver = str(rdata)
@@ -374,7 +284,7 @@ class BaseMigrator():
Adds an MX to GNS
"""
dnsname, ttl, rdata = record
- if not BaseMigrator.check_if_record_exists_in_zone(record, zonename):
+ if not GNSMigrator.check_if_record_exists_in_zone(record, zonename):
rdatalist = str(rdata).split(' ')
value = '%s,%s' % (rdatalist[0], rdatalist[1])
sp.run([GNUNET_NAMESTORE_COMMAND,
@@ -417,7 +327,7 @@ class BaseMigrator():
try:
soa_answer = dns.resolver.query(domain, 'SOA')
except dns.resolver.NoAnswer:
- print("the domain '%s' does not exist" % domain)
+ logging.warning("the domain '%s' does not exist")
master_answer = dns.resolver.query(soa_answer[0].mname, 'A')
try:
if resolver:
@@ -428,10 +338,16 @@ class BaseMigrator():
master_answer[0].address, domain,
port=cls.port))
except dns.resolver.NoAnswer:
- print("nameserver for '%s' did not answer" % domain)
+ logging.warning("nameserver for '%s' did not answer", domain)
except dns.exception.FormError:
- print("domain '%s' does not allow xfr requests" % domain)
+ logging.warning("domain '%s' does not allow xfr requests", domain)
for soa_record in zone.iterate_rdatas(rdtype=dns.rdatatype.SOA):
+ if not cls.transferns:
+ mname = soa_record[2].mname
+ if cls.domain not in mname:
+ cls.transferns = str(soa_record[2].mname) + "." + domain
+ else:
+ cls.transferns = str(soa_record[2].mname)
return soa_record[2].serial
@staticmethod
@@ -456,56 +372,6 @@ class BaseMigrator():
soa_serial = 0
return soa_serial
-
-class ZoneMigrator(BaseMigrator):
- """
- Class that migrates small zones efficiently
- """
- @classmethod
- def __init__(cls, domainlist, port=53):
- BaseMigrator.__init__(domainlist, port)
-
-class TLDMigrator(BaseMigrator):
- """
- Class that migrates big zones (TLDs) efficiently
- """
- @classmethod
- def __init__(cls, tld, transferns, port=53):
- BaseMigrator.__init__(tld, port)
- cls.soa = None
- cls.tld = tld
- cls.transferns = transferns
- cls.zone = None
- cls.zonegenerator = {}
-
- @classmethod
- def initial_zone_transfer(cls, serial=None):
- """
- Transfer and initialize the zone
- """
- if serial:
- cls.zonegenerator = dns.query.xfr(cls.transferns,
- cls.tld,
- rdtype=dns.rdatatype.IXFR,
- serial=serial,
- port=cls.port)
- else:
- cls.zonegenerator = dns.query.xfr(cls.transferns, cls.tld,)
-
- @classmethod
- def bootstrap_zone(cls):
- """
- Creates the zone in gnunet
- """
- reverse_parsing = cls.tld.split('.')[::-1]
- reverse_parsing = list(filter(None, reverse_parsing))
- for domainpart in reverse_parsing:
- try:
- sp.run([GNUNET_ZONE_CREATION_COMMAND,
- '-C', domainpart])
- except sp.CalledProcessError:
- print("Zone %s already exists!" % domainpart)
-
@staticmethod
def get_zone_soa(zone):
"""
@@ -523,70 +389,36 @@ class TLDMigrator(BaseMigrator):
"""
Extract necessary information from Generator
"""
- currentserial = int(cls.get_current_serial(cls.tld, cls.transferns))
- zoneserial = int(cls.get_zone_serial(cls.tld[:-1]))
- print(zoneserial)
- print(currentserial)
+ currentserial = int(cls.get_current_serial(cls.domain, cls.transferns))
+ zoneserial = int(cls.get_zone_serial(cls.domain[:-1]))
if zoneserial == 0:
cls.initial_zone_transfer()
- cls.transfer_zone()
+ cls.zone = dns.zone.from_xfr(cls.zonegenerator)
cls.soa = cls.get_zone_soa(cls.zone)
elif zoneserial < currentserial:
cls.initial_zone_transfer(serial=zoneserial)
- cls.transfer_zone()
+ cls.zone = dns.zone.from_xfr(cls.zonegenerator)
cls.soa = cls.get_zone_soa(cls.zone)
+ return
elif zoneserial == currentserial:
- print("Nothing to do!")
+ logging.warning("Nothing to do!")
sys.exit(0)
# should be unnecessary but AXFR SOA is not equal to direct SOA
else:
- print("SOA serial is bigger than zone serial?")
+ logging.critical("SOA serial is bigger than zone serial?")
print(zoneserial, currentserial)
- sys.exit(0)
-
- @classmethod
- def transfer_zone(cls, zone_factory=dns.zone.Zone, relativize=True):
- """
- Do the actual zone transfer
- """
- zone = None
- try:
- for message in cls.zonegenerator:
- origin = message.origin
- rdclass = message.answer[0].rdclass
- if zone is None:
- if relativize:
- origin = message.origin
- else:
- origin = message.answer[0].name
- rdclass = message.answer[0].rdclass
- zone = zone_factory(origin, rdclass, relativize=relativize)
- for rrset in message.answer:
- znode = zone.nodes.get(rrset.name)
- if not znode:
- znode = zone.node_factory()
- zone.nodes[rrset.name] = znode
- zrds = znode.find_rdataset(rrset.rdclass, rrset.rdtype,
- rrset.covers, True)
- zrds.update_ttl(rrset.ttl)
- for record in rrset:
- record.choose_relativity(zone.origin, relativize)
- zrds.add(record)
- cls.zone = zone
- except Exception as transferexception:
- print("Error occured during Zone transfer: %s" % transferexception)
- raise
+ sys.exit(1)
@classmethod
- def multithreaded_add_records_to_gns(cls):
+ def add_records_to_gns(cls):
"""
Extracts records from zone and adds them to GNS
"""
- print("Starting to add records into GNS...")
- zonename = cls.tld.split('.')[0]
+ logging.info("Starting to add records into GNS...")
+ zonename = cls.get_lowest_domain_part(cls.domain)
# Defining FIFO Queue
- taskqueue = queue.Queue(maxsize=200)
+ taskqueue = queue.Queue(maxsize=5)
# Defining worker
def worker():
while True:
@@ -602,34 +434,29 @@ class TLDMigrator(BaseMigrator):
# building gns record struct
# GNUnetGNSRecordData()
- cls.add_record_to_gns(record, zonename, cls.tld)
+ cls.add_record_to_gns(record, zonename, cls.domain)
taskqueue.task_done()
- # Create threads
- threads = []
- for _ in range(multiprocessing.cpu_count()):
- thread = threading.Thread(target=worker)
- thread.start()
- threads.append(thread)
+ # Create one thread
+ thread = threading.Thread(target=worker)
+ thread.start()
- # Give workers stuff to do
- for record in cls.zone.iterate_rdatas(rdtype=dns.rdatatype.NS):
+ # Give worker stuff to do
+ for record in cls.zone.iterate_rdatas():
taskqueue.put(record)
# Block until all tasks are done
taskqueue.join()
# Stop workers
- for _ in range(multiprocessing.cpu_count()):
- taskqueue.put(None)
- for thread in threads:
- thread.join()
+ taskqueue.put(None)
+ thread.join()
- # Add soa record to GNS once completed
+ # Add soa record to GNS once completed (updates the previous one)
soa = cls.get_zone_soa(cls.zone)
- super().add_soa_record_to_gns(soa, cls.tld[:-1], cls.tld)
- print("All records have been added!")
+ cls.add_record_to_gns(soa, zonename, cls.domain)
+ logging.info("All records have been added!")
@staticmethod
def add_record_to_gns(record, zonename, domain):
@@ -641,22 +468,23 @@ class TLDMigrator(BaseMigrator):
"""
dnsname_str = str(record[0])
rtype_str = dns.rdatatype.to_text(record[2].rdtype)
+ logging.info("adding %s record with name %s", rtype_str, dnsname_str)
if dnsname_str == '@':
if rtype_str == 'SOA':
- BaseMigrator.add_soa_record_to_gns(record, zonename, domain)
+ GNSMigrator.add_soa_record_to_gns(record, zonename, domain)
else:
if rtype_str == 'NS' and dnsname_str != '@':
- BaseMigrator.add_ns_record_to_gns(record, zonename, domain)
+ GNSMigrator.add_ns_record_to_gns(record, zonename, domain)
elif rtype_str == 'MX':
- BaseMigrator.add_mx_record_to_gns(record, zonename)
+ GNSMigrator.add_mx_record_to_gns(record, zonename)
#elif rtype_str == 'SRV':
- # BaseMigrator.add_srv_record_to_gns(record, zonename)
+ # GNSMigrator.add_srv_record_to_gns(record, zonename)
elif rtype_str in ['A', 'AAAA']:
- BaseMigrator.add_a_aaaa_record_to_gns(record, zonename, domain)
+ GNSMigrator.add_a_aaaa_record_to_gns(record, zonename, domain)
elif rtype_str in ['TXT', 'CNAME']:
- BaseMigrator.add_gen_record_to_gns(record, zonename)
+ GNSMigrator.add_gen_record_to_gns(record, zonename)
else:
- print("Record type %s is not yet supported" % rtype_str)
+ logging.warning("Record type %s is not yet supported", rtype_str)
def main():
@@ -664,39 +492,32 @@ def main():
Initializes object and handles arguments
"""
# argument parsing from docstring definition
- args = docopt.docopt(__doc__, version='GNS Migrator 0.1.3')
+ args = docopt.docopt(__doc__, version='GNS Migrator 0.2.0')
# Checks if GNUnet services are running
try:
sp.check_output([GNUNET_ARM_COMMAND, '-I'], timeout=1)
except sp.TimeoutExpired:
- print('GNUnet Services are not running!')
- print('Exiting...')
+ logging.critical('GNUnet Services are not running!')
sys.exit(1)
- tld = args.get('<tld>', None)
- transferns = args.get('<transferns>', None)
- txtfile = args.get('<txtfile>', None)
- port = int(args.get('<port>', None))
-
- if tld and transferns:
- migrator = TLDMigrator(tld, transferns, port)
- serial = migrator.get_zone_serial(tld)
- migrator.initial_zone_transfer(serial)
- migrator.bootstrap_zone()
- migrator.mirror_zone()
- migrator.multithreaded_add_records_to_gns()
- elif txtfile:
- domainlist = []
- txtfile = args['<txtfile>']
- with open(txtfile, 'r') as openedtxt:
- for line in openedtxt:
- domainlist.append(line.rstrip())
- zonemigrator = ZoneMigrator(domainlist, port=port)
- zonemigrator.initial_zone_transfer(resolver=transferns)
- zonemigrator.bootstrap_zones()
- for domain, zonetuple in zonemigrator.zones.items():
- zonemigrator.refresh_zone(domain, zonetuple)
+ # Not ideal as this will always be
+ debug = args['--debug']
+ domain = args.get('<domain>', None)
+ transferns = args['<transferns>'] if args['<transferns>'] else None
+ port = args['<port>'] if args['<port>'] else 53
+
+ # Change logging severity to debug
+ if debug:
+ logging.basicConfig(level=logging.DEBUG)
+
+ # Initialize class instance
+ migrator = GNSMigrator(domain, transferns, port)
+ serial = migrator.get_zone_serial(domain)
+ migrator.initial_zone_transfer(serial)
+ migrator.bootstrap_zone()
+ migrator.mirror_zone()
+ migrator.add_records_to_gns()
if __name__ == '__main__':
main()
