Source code for cncopgetprofile

#!/usr/bin/env python3
# vim: set fileencoding=utf-8
# pylint:disable=line-too-long
r""":mod:`cncopgetprofile` - download jobs

.. module:: cncopgetprofile
   :synopsis: download jobs
.. moduleauthor:: Jim Carroll <>
.. image:: ../appicons/cncopgetprofile.png


Connect to the cncop network and download the backup jobs assigned to this
device. Also check for software updates, and if updates exist, notify the
:mod:`ccs3updater` module.

It's intended that the ``cncopgetprofile`` module be run hourly to check
for changes to the backup jobs assigned to this device. The changes are
download from the cncop network and stored locally in
:file:`Config/backupjobs.db` database.

Software updates

With each connection to the cncop network, the ``cncopgetprofile`` module also
checks for software updates. If any updates exists, a message is sent to the
:mod:`ccs3updater` process which is listening on UDP port 9998.

The auto software update process can be disabled in the
:file:`Config/cncop3.ini`, by setting ``autoupdate`` to Off in the
``[version]`` section.

Open firewall request

Carroll-Net's network employs a strict set of firewall policies.  Occasionally,
these policies interfere with a registered device performing backup services.
To detect and report when this happens, ``cncopgetprofile`` monitors policy
changes, and reports unintentional blockages to the cncop network.

Command line options

*usage:* `` [-?] [-d] [-c CONFIG] [--noupdate] [--update]

Optional argument:

.. option:: ?, -h, --help

   Display help and exit

.. option:: -d, --debug

   Generate diagnostic logging. The output is directed to the
   :file:`Spool/Logs` folder and can be viewed using the cncop watcher

.. option:: -c CONFIG, --config CONFIG

   Alternate configuration file.

.. option:: --noupdate

   Disable the check for software updates.

.. option:: --update

   Force a check for software update, even if autoupdate is disabled in the
   :file:`Config/cncop.ini` files.

.. option:: --recreate

   Force backupjobs database to be recreated. Typically used to deploy schema

   Copyright(c) 2013, Carroll-Net, Inc., All Rights Reserved"""
# pylint:enable=line-too-long
# ----------------------------------------------------------------------------
# Standard library imports
# ----------------------------------------------------------------------------
import argparse
import io
import datetime
import http.client
import logging
import logging.config
import os
import socket
import subprocess
import sys
import tempfile
import urllib
import xml.etree.ElementTree as ET  # nosec

# ----------------------------------------------------------------------------
# Project imports
# ----------------------------------------------------------------------------
import cncop.identity
import cncop.mutex
import cncop.process
import cncop.profile
import cncop.settings
import cncop.utils
import cncop.web
import lib.backupjobdb
import lib.clogging
import lib.utils

# ----------------------------------------------------------------------------
# Module level initializations
# ----------------------------------------------------------------------------
__version__ = '3.0.1'
__author__ = 'Jim Carroll'
__email__ = ''
__status__ = 'Production'
__copyright__ = 'Copyright(c) 2013, Carroll-Net, Inc., All Rights Reserved'
__icon__ = 'appicons/cncopgetprofile.ico'

AGENT = {'name': 'cncopgetprofile',
         'copyright': __copyright__,
         'version': __version__}
LOG = logging.getLogger(AGENT['name'])
SPC9 = ' ' * 9
ON_POSIX = 'posix' in sys.builtin_module_names

[docs]def run_cncopreporter(): r"""Run cncopreporter to upload reports to cncop network""" app = lib.utils.appname('cncopreporter') cmd = [sys.executable, app] if app.endswith('.py') else [app] with subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=0, close_fds=ON_POSIX) as task: return cncop.process.drain_output([task])[0]
[docs]class CncOpGetProfile: r"""Retrieve the profile for this device""" def __init__(self, args): self.args = args cncop.settings.set_config_file(args.config) ppath = cncop.settings.profile_path() try: kfname = os.path.join(ppath, 'identity.key') kfile = cncop.identity.KeyFile(kfname) pubkey = kfile.PublicKey() except OSError as exc: raise RuntimeError('Missing identity file %s: %s' % (kfname, exc)) from None self.profdb = cncop.profile.ProfileDb() self.device = self.profdb.find_device_by_pubkey(pubkey) if not self.device: LOG.error(">>> ERROR: Could not find registered profile") LOG.error(">>> Consider running ''") raise RuntimeError('Missing profile.db')'>>> %s Device role %s', EPFX, self.device.role) self.web = cncop.web.Client(keyfile=kfile)
[docs] def check_firewall(self): r"""Called to check/request firewall be opened""" # Re-try the connect -- but we ONLY care about timeouts (anything other # than tmo is something other than f/w) sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.settimeout(5.0) try: sock.connect((, 443)) except socket.timeout: # Likely f/w block (or local internet outage) pass except Exception as exc: # pylint: disable=broad-except LOG.error('Unexpected error: %s', str(exc)) return org = self.profdb.get_organization(self.device.orgid) url = ('/openfirewall/?' 'orgname=%s&' 'orgid=%s&' 'devicename=%s&' 'deviceid=%s' % ( urllib.parse.quote(org.orgname), urllib.parse.quote(str(org.orgid)), urllib.parse.quote(self.device.devicename), urllib.parse.quote(str(self.device.deviceid)) )) doc = ET.Element('openfirewallrequest') doc.append(self.device.to_xml()) msg = io.BytesIO() msg.write(b"<?xml version='1.0' encoding='utf-8'>\n") msg.write(ET.tostring(doc)) body = msg.getvalue() msg.close() try: conn = http.client.HTTPConnection('', timeout=60.0) conn.request('POST', url, body) conn.getresponse() conn.close() except Exception as exc: # pylint: disable=broad-except LOG.error(str(exc))
[docs] def update_sw(self): r"""Check whether there is new software for us""" if not cncop.settings.get_autoupdate() and not self.args.update:'>>> %s Autoupdate disabled, skip update_sw check', EPFX) return try: doc = self.web.get_latest_sw_version(self.device) LOG.debug('%s', ET.tostring(doc)) except cncop.web.ProtocolError as exc: LOG.warning('>>> %s WARNING No software available device.', EPFX) LOG.debug('%s', exc) return swdoc = doc.find('software') availableversion = swdoc.find('version').text if availableversion == cncop.settings.get_currentversion():'%sAlready up to date', SPC9) return'>>> %s New version %s available. Requesting update', EPFX, availableversion) msg = ET.tostring(doc) # Let ccs3updater know there is updated software to d/l sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) sock.sendto(msg, ('', NOTIFY_PORT))
[docs] def update_jobs_db(self, doc): r"""Extract jobs from xml doc, store in jobdb"""'>>> %s Save backupjobs, started %s', EPFX, lib.utils.utcstr()) fdn, tmpf = tempfile.mkstemp() os.close(fdn) with lib.backupjobdb.JobDb({'recreate': True, 'dsn': 'sqlite:///%s' % tmpf}) as tmpdb: for job in doc.findall('backupjob'): buj = lib.backupjobdb.BackupJob.fromxml(job) LOG.debug('%sAdd Job "%s"', SPC9, buj) LOG.debug('%sDetails: %s', SPC9, buj.tostring()) tmpdb.session.add(buj) for filt in job.findall('filters/filter'): frec = lib.backupjobdb.Filter.fromxml(filt) LOG.debug('%sAdd Filter "%s"', SPC9, filt) LOG.debug('%sDetails: %s', SPC9, frec.tostring()) tmpdb.session.add(frec) tmpdb.session.flush() # Connect to main job's database and delete jobs & filters with lib.backupjobdb.JobDb({'recreate': self.args.recreate}) \ as jobdb: jobdb.session.query(lib.backupjobdb.BackupJob).delete() jobdb.session.query(lib.backupjobdb.Filter).delete() # Add downloaded jobs and filters for job in tmpdb.findall_backupjobs(): jobdb.session.merge(job) for filt in tmpdb.find_filters(job.jobid): jobdb.session.merge(filt) os.unlink(tmpf)
[docs] def Run(self): r"""application driver""" start = datetime.datetime.utcnow()'>>> %s Download Jobs, started %s', EPFX, lib.utils.utcstr()) try: doc = self.web.get_device_jobs(self.device) LOG.debug('%s', ET.tostring(doc, 'utf-8')) except cncop.web.NetworkingError as exc: LOG.error(">>> %s Network error, checking firewall", EPFX) return self.check_firewall() # Update the jobs database self.update_jobs_db(doc) # Run cncopreporter (to force delivery of cached logs)'>>> %s Run cncopreporter, started %s', EPFX, lib.utils.utcstr()) run_cncopreporter() # Check for s/w updates if not self.args.noupdate:'>>> %s Check for S/w updates, started %s', EPFX, lib.utils.utcstr()) self.update_sw()'>>> %s Runtime %s', EPFX, datetime.datetime.utcnow() - start) return 0
[docs]def main(): r"""process main driver""" # pylint: disable=too-many-return-statements orig_config = cncop.settings.config_file() parser = argparse.ArgumentParser( add_help=False, description='Download jobs from cncop network.') parser.add_argument('-?', '-h', '--help', dest='help', action='store_true', default=False, help='Show this help message and exit') parser.add_argument('-d', '--debug', action='store_true', help='Increase logging level to DEBUG') parser.add_argument('-c', '--config', default=orig_config, help='Alternate configuration file.') parser.add_argument('--noupdate', action='store_true', help='Disable software updates') parser.add_argument('--update', action='store_true', help='Force check software update (even if disabled)') parser.add_argument('--recreate', action='store_true', help='Force backupjobs db to be recreated ' '(Used to change db schema).') args = parser.parse_args() if parser.print_help() return 0 cncop.settings.set_config_file(args.config) mutex = cncop.mutex.Mutex('%s.lock' % AGENT['name']) if not mutex.tryacquire(): print('>>> %s Another copy of %s is running.' % (EPFX, AGENT['name'])) return 0 if (result := lib.clogging.activate(AGENT, EPFX)): return result if args.debug: LOG.setLevel(logging.DEBUG)'>>> %s Option: Debug enabled', EPFX) if args.config != orig_config:">>> %s Option: Config set to '%s'", EPFX, args.config) if args.noupdate and args.update: LOG.error('>>> %s ERROR: Cannot combine --noupdate and --update', EPFX) return -1 if args.noupdate:'>>> %s Option: Disable software updates', EPFX) if args.update:'>>> %s Option: Force a check for software updates', EPFX) if args.recreate:'>>> %s Option: Recreate backupjobs.db', EPFX) try: return CncOpGetProfile(args).Run() except KeyboardInterrupt:'>>> %s Terminated by CTRL-C', EPFX) except RuntimeError as exc: LOG.error('>>> %s Terminated due to %s', EPFX, exc) return -1
if __name__ == '__main__': sys.exit(main())