def __get_status(self, server, user, user_ip, req_id, start=0, count=100):
        try:
            (host, port) = server.split(':')
            port = int(port)

        except ValueError:
            logs.error("invalid server address in network XML: %s" % server)
            raise wsgicomm.WIInternalError, "invalid server address"

        try:
            arcl = Arclink()
            arcl.open_connection(host, port, user, user_ip=user_ip,
                timeout=self.status_timeout)

            try:
                #new version requires an arclink update to support pagination
                #status = arcl.get_status(req_id, start, count)
                status = arcl.get_status(req_id)
                status.request = status.request[:count]
                return status

            finally:
                arcl.close_connection()

        except (ArclinkError, socket.error) as e:
            raise wsgicomm.WIServiceError, str(e)

def parseDate(datestr):
    m = _rx_datetime.match(datestr)
    if not m:
        logs.error("invalid date: " + datestr)
        return (seiscomp3.Core.Time(1980, 1, 1, 0, 0, 0), "1980-01-01T00:00:00.0000Z")

    try:
        year = int(m.group("year"))
        (month, mday) = _dy2mdy(int(m.group("doy")), year)

        if m.group("hour"):
            hour = int(m.group("hour"))
            minute = int(m.group("minute"))
        else:
            hour = 0
            minute = 0

        if m.group("second"):
            second = int(m.group("second"))
        else:
            second = 0

        coretime = seiscomp3.Core.Time(year, month, mday, hour, minute, second)

    except (TypeError, ValueError, IndexError):
        logs.error("invalid date: " + datestr)
        return (seiscomp3.Core.Time(1980, 1, 1, 0, 0, 0), "1980-01-01T00:00:00.0000Z")

    return (coretime, coretime.toString("%Y-%m-%dT%H:%M:%S.%fZ"))

def loadGains(fileName):
    try:
        fd = open(fileName)
        try:
            line = fd.readline()
            lineno = 0
            while line:
                line = line.strip()
                lineno += 1

                if not line or line[0] == '#':
                    line = fd.readline()
                    continue

                try:
                    (deviceName, deviceIdPattern, streamPattern, gain) = line.split()
                    if deviceName in gainTable:
                        gainTable[deviceName].append((streamPattern, deviceIdPattern,
                            float(gain)))
                    else:
                        gainTable[deviceName] = [ (streamPattern, deviceIdPattern,
                            float(gain)) ]

                except (TypeError, ValueError):
                    logs.error("%s:%d: parse error" % (fileName, lineno))
                    sys.exit(1)

                line = fd.readline()

        finally:
            fd.close()

    except IOError, e:
        logs.error("cannot open %s: %s" % (fileName, str(e)))
        sys.exit(1)

def show_status(request):
    try:
        logs.info(" datacenter name: " + request.dcname)
        rqstat = request.status()
    except ArclinkError, e:
        logs.error(str(e))
        return

    def scan_cha(d):
        last_file = {}

        for f in os.listdir(d):
            try:
                (net, sta, loc, cha, ext, year, doy) = f.split('.')
                nets.add((net, int(year)))

            except ValueError:
                logs.error("invalid SDS file:" + p, True)
                continue

            if (net, sta, loc, cha) not in timespan:
                continue

            try:
                if doy > last_file[loc][0]:
                    last_file[loc] = (doy, f)

            except KeyError:
                last_file[loc] = (doy, f)

        for (loc, (doy, f)) in last_file.items():
            with open(d + '/' + f, 'rb') as fd:
                nslc = tuple(f.split('.')[:4])
                rec = mseedlite.Record(fd)
                fd.seek(-rec.size, 2)
                rec = mseedlite.Record(fd)
                ts = timespan[nslc]

                if ts.start < rec.end_time < ts.end:
                    ts.start = rec.end_time

                elif rec.end_time >= ts.end:
                    del timespan[nslc]

def parseOrientation(orientation):
    for x in orientation.split(';'):
        try:
            (code, azimuth, dip) = x.split()
            yield (code, float(azimuth), float(dip))
        except (TypeError, ValueError):
            logs.error("error parsing orientation %s at %s" % (orientation, x))
            continue

def main():
    (SSLpasswordDict, addr, request_format, data_format, label, resp_dict, rebuild_volume, proxymode, user, timeout, retries, output_file, input_file, spfr) = process_options()

    try:
        ret = _main(SSLpasswordDict, addr, request_format, data_format, label, resp_dict, rebuild_volume, proxymode, user, timeout, retries, output_file, input_file, spfr)

    except ArclinkError, e:
        logs.error(str(e))
        ret = 1

def parse_breqfast_from_handler(req, fh):
    parser = BreqParser()
    parser.parse_email_from_handler(fh)
    req.content = parser.reqlist
    logs.debug("")
    if parser.failstr:
        logs.error(parser.failstr)
    else:
        logs.info("parsed %d lines from breqfast message" % len(req.content))

    def next(self):
        while True:
            try:
                return Record(self.__fd)

            except MSeedNoData:
                pass

            except MSeedError, e:
                logs.error(str(e))

    def __parse_arglist(self, arglist):
        d = {}
        for arg in arglist:
            pv = arg.split('=', 1)
            if len(pv) != 2:
                logs.error("invalid request args in status: " + args)
                continue

            d[pv[0]] = pv[1]

        return d

    def getConfigFloat(self, name, default = None):
        try:
            return float(self.__cfg.getString(name))

        except ValueError:
            logs.error("config parameter '%s' has invalid value" % name)
            return default

        #except seiscomp3.Config.OptionNotFoundException:
        except Exception, e:
            return default

def parse_native_from_handler(req, fd):
    rqline = fd.readline()
    while rqline:
        rqline = rqline.strip()
        if not rqline:
            rqline = fd.readline()
            logs.debug("skipping empty request line")
            continue
            
        rqsplit = rqline.split()
        if len(rqsplit) < 3:
            logs.error("invalid request line: '%s'" % (rqline,))
            rqline = fd.readline()
            continue

        try:
            start_time = datetime.datetime(*map(int, rqsplit[0].split(",")))
            end_time = datetime.datetime(*map(int, rqsplit[1].split(",")))
        except ValueError, e:
            logs.error("syntax error (%s): '%s'" % (str(e), rqline))
            rqline = fd.readline()
            continue

        network = rqsplit[2]
        station = "."
        channel = "."
        location = "."

        i = 3
        if len(rqsplit) > 3 and rqsplit[3] != ".":
            station = rqsplit[3]
            i += 1
            if len(rqsplit) > 4 and rqsplit[4] != ".":
                channel = rqsplit[4]
                i += 1
                if len(rqsplit) > 5 and rqsplit[5] != ".":
                    location = rqsplit[5]
                    i += 1
                    
        while len(rqsplit) > i and rqsplit[i] == ".":
            i += 1
        
        constraints = {}
        for arg in rqsplit[i:]:
            pv = arg.split('=', 1)
            if len(pv) != 2:
                raise ArclinkError, "invalid request syntax"
            
            constraints[pv[0]] = pv[1]

        req.add(network, station, channel, location, start_time, end_time,
            constraints)
        
        rqline = fd.readline()

    def request_purge(self, envir, params):
        """Delete one user request at a given server.

        Input:  server          server DCID
                user            user ID
                request         request ID

        Output: true

        """
        dcid = params.get("server")
        user = params.get("user")
        req_id = params.get("request")

        if dcid is None:
            raise wsgicomm.WIClientError, "missing server"

        else:
            try:
                server = self.nodes[dcid]['address']

            except KeyError:
                raise wsgicomm.WIClientError, "invalid server"

        if user is None:
            raise wsgicomm.WIClientError, "missing user ID"

        if req_id is None:
            raise wsgicomm.WIClientError, "missing request ID"

        try:
            (host, port) = server.split(':')
            port = int(port)

        except ValueError:
            logs.error("invalid server address in network XML: %s" % server)
            raise wsgicomm.WIInternalError, "invalid server address"

        try:
            arcl = Arclink()
            arcl.open_connection(host, port, user)

        except (ArclinkError, socket.error) as e:
            raise wsgicomm.WIServiceError, str(e)

        try:
            arcl.purge(req_id)
            return json.dumps(True)

        except (ArclinkError, socket.error) as e:
            arcl.close_connection()
            raise wsgicomm.WIServiceError, str(e)

    def log_print(s):
        if logstream:
            logstream.write(s + "\n")

        else:
            try:
                fh = file(fname, "a")
                try:
                    fh.write(s + "\n")
                finally:
                    fh.close()
            except OSError:
                logs.error("Log file %s could not be opened!" % fname)

def parsePkgstr(pkgstr):
    result = {}
    for x in pkgstr.split():
        m = _rx_pkg.match(x)
        if not m:
            logs.error("error parsing %s at %s" % (pkgstr, x))
            continue

        result[m.group('pkg')] = m.group('profile')

    if 'trunk' not in result:
        result['trunk'] = None

    return result

    def run(self):
        try:
            if self.dcid is None:
                print >>sys.stderr, "Please specify datacenter/archive ID"
                return False

            nettab = Nettab(self.dcid)
            instdb = Instruments(self.dcid)

            try:
                self.__load_file(instdb.load_db, self.inst_db_file)
                self.__load_file(nettab.load_statmap, self.stat_map_file)
                self.__load_file(nettab.load_access_net, self.access_net_file)
                self.__load_file(nettab.load_access_stat, self.access_stat_file)
                self.__load_file(instdb.load_sensor_attr, self.sensor_attr_file)
                self.__load_file(instdb.load_datalogger_attr, self.datalogger_attr_file)
                self.__load_file(nettab.load_network_attr, self.network_attr_file)
                self.__load_file(nettab.load_station_attr, self.station_attr_file)

                inv = SC3Inventory(DataModel.Inventory())

                idx = 1
                for tab in sorted(self.tab_files):
                    print >>sys.stderr, "Loading %s (%d/%d)" % (tab, idx, len(self.tab_files))
                    self.__load_file(nettab.load_tab, tab)
                    print >>sys.stderr, "Generating data structures"
                    nettab.update_inventory(instdb, inv)
                    idx = idx + 1
                    if self.isExitRequested():
                        print >>sys.stderr, "Exit requested, abort"
                        return False

                print >>sys.stderr, "Generating output"
                ar = IO.XMLArchive()
                ar.setFormattedOutput(self.commandline().hasOption("formatted"))
                ar.create(self.out_file)
                ar.writeObject(inv.obj)
                ar.close()
                print >>sys.stderr, "Finished"

            except (IOError, NettabError), e:
                logs.error("fatal error: " + str(e))
                return False

        except Exception:
            logs.print_exc()
            return False

        return True

    def __load_module(self, path):
        modname = os.path.splitext(os.path.basename(path))[0].replace('.', '_')

        if modname in self.__modules:
            logs.error("'%s' is already loaded!" % modname)
            return

        try:
            mod = imp.load_source('__wi_' + modname, path)

        except:
            logs.error("Error loading '%s'" % modname)
            logs.print_exc()
            return

        self.__modules[modname] = mod.WI_Module(self)

    def getStreamInfo(self, start_time, end_time, net, sta, cha, loc):
        try:
            stream_epochs = self.streamidx[(net, sta, cha, loc)]
        except KeyError:
            logs.error("%s,%s,%s,%s not found" % (net, sta, cha, loc))
            return None

        for stream in stream_epochs:
            try:
                station = self.stations[self.sensorsLoc[stream[0]][0]]

            except IndexError:
                logs.error("cache inconsistency")
                return None

            # stream_start = datetime.datetime(station[8], 1, 1)
            # stream_end = datetime.datetime(station[9], 1, 1) if station[9] \
            #         else datetime.datetime(2030, 1, 1)
            stream_start = stream[6]
            stream_end = stream[7] if stream[7] is not None \
                else (datetime.datetime.now() + datetime.timedelta(days=365))

            if start_time >= stream_end or end_time <= stream_start:
                continue

            result = {'latitude': station[5],
                      'longitude': station[6],
                      'elevation': station[10]}

            if stream[3] != 0:
                tdiff = end_time - start_time
                tdiff = tdiff.days * 86400 + tdiff.seconds
                samp = float(stream[4]) / float(stream[3])

                # assuming approximately 1 byte per sample (compressed),
                # 512 bytes record size
                bytesper = 1
                recsize = 512
                result['size'] = int(recsize * math.ceil(
                                     float(tdiff * samp * bytesper) / recsize))

            else:
                result['size'] = 0

            return result

        return None

    def close(self):
        try:
            try:
                seed_volume = SEEDVolume(self.__inv, ORGANIZATION, LABEL, self.__resp_dict)

                self.__mseed_fd.seek(0)
                for rec in mseed.Input(self.__mseed_fd):
                    seed_volume.add_data(rec)

                seed_volume.output(self.__fd)

            except (mseed.MSeedError, SEEDError, DBError), e:
                logs.error("error creating SEED volume: " + str(e))

        finally:
            self.__mseed_fd.close()
            self.__fd.close()

    def run(self):
        try:
            seiscompRoot = self.commandline().unrecognizedOptions()[0]
            sys.stderr.write("root directory: %s\n" % seiscompRoot)

            try:
                DCID = self.configGetString("datacenterID")

            except:
                logs.error("datacenterID not found in global.cfg")
                return False

            networkRestricted = {}
            incompleteResponse = {}

            global instdb
            instdb = Instruments(DCID)

            self.__load_file(loadGains, os.path.join(seiscompRoot, "config", "gain.dlsv"))

            # for backwards compatibility
            self.__load_file(loadGains, os.path.join(seiscompRoot, "config", "gain.tab.out"))
            self.__load_file(loadGains, os.path.join(seiscompRoot, "config", "gain.tab"))

            try:
                self.__load_file(instdb.load_db, os.path.join(seiscompRoot, "resp", "inst.db"))
                self.__load_file(instdb.load_sensor_attr, os.path.join(seiscompRoot, "resp", "sensor_attr.csv"))
                self.__load_file(instdb.load_datalogger_attr, os.path.join(seiscompRoot, "resp", "datalogger_attr.csv"))

            except (IOError, NettabError), e:
                logs.error("fatal error: " + str(e))
                return False

            sc3Inv = seiscomp3.DataModel.Inventory()
            inventory = InventoryWrapper(sc3Inv, DCID)

            existingNetworks = set()
            existingStations = set()

            for f in glob.glob(os.path.join(seiscompRoot, "key", "network_*")):
                try:
                    logs.debug("processing " + f)
                    netCode = f.split("/network_")[-1]
                    try:
                        kf = Keyfile(f)
                    except IOError, e:
                        logs.error(str(e))
                        continue

                    existingNetworks.add(netCode)
                    networkRestricted[netCode] = False

                    inventory.updateNetwork(netCode, kf)

                except ValueError, e:
                    logs.error("%s: %s" % (f, str(e)))