Logo Search packages:      
Sourcecode: leafnode version File versions

fetchnews.c

/*
fetchnews -- post articles to and get news from upstream server(s)

Written by Arnt Gulbrandsen <agulbra@troll.no> and copyright 1995
Troll Tech AS, Postboks 6133 Etterstad, 0602 Oslo, Norway, fax +47
22646949.
Modified by Cornelius Krasel <krasel@wpxx02.toxi.uni-wuerzburg.de>
and Randolf Skerka <Randolf.Skerka@gmx.de>.
Copyright of the modifications 1997.
Modified by Kent Robotti <robotti@erols.com>. Copyright of the
modifications 1998.
Modified by Markus Enzenberger <enz@cip.physik.uni-muenchen.de>.
Copyright of the modifications 1998.
Modified by Cornelius Krasel <krasel@wpxx02.toxi.uni-wuerzburg.de>.
Copyright of the modifications 1998, 1999.
Modified by Ralf Wildenhues <ralf.wildenhues@gmx.de>.
Copyright of the modifications 2002.
Modified by Jonathan Larmour <jifl@jifvik.org>.
Copyright of the modifications 2002.
Modified by Richard van der Hoff <richard@rvanderhoff.org.uk>
Copyright of the modifications 2002.
Enhanced and modified by Matthias Andree <matthias.andree@web.de>.
Copyright of the modifications 2000 - 2003.

See file COPYING for restrictions on the use of this software.
*/

#include "leafnode.h"
#include "fetchnews.h"
#include "mastring.h"
#include "ln_log.h"
#include "mysigact.h"

#include <sys/types.h>
#include <ctype.h>
#include "system.h"
#include <fcntl.h>
#include <netdb.h>
#include <netinet/in.h>
#include <setjmp.h>
#include <signal.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/socket.h>
#include <sys/stat.h>
#include <syslog.h>
#include <sys/resource.h>
#include <unistd.h>
#include <utime.h>
#include <sys/wait.h>

#include "groupselect.h"

int verbose = 0;
int debug = 0;

static time_t now;

/* Variables set by command-line options which are specific for fetch */
static unsigned long extraarticles = 0;
static int usesupplement = 1;
static int postonly = 0;      /* if 1, don't read files from upstream */
static int noexpire = 0;      /* if 1, don't automatically unsubscribe */
static int forceactive = 0;   /* if 1, reread complete active file */

static sigjmp_buf jmpbuffer;
static volatile sig_atomic_t canjump;

static int age( /*@null@*/ const char *date);
static int postarticles(const struct server *current_server);

static void
ignore_answer(FILE * f)
{
    char *l;
    while (((l = mgetaline(f)) != NULL) && strcmp(l, "."));
}

static RETSIGTYPE
sig_int(int signo)
{
    if (canjump == 0)
      return;           /* ignore unexpected signals */
    if (signo == SIGINT || signo == SIGTERM) {
      canjump = 0;
      alarm(0);
      siglongjmp(jmpbuffer, signo);
    }
}

static void
usage(void)
{
    fprintf(stderr, "Usage: fetchnews [-q] [-v] [-x #] [-l] [-n] [-f] [-P] [-w]\n"
          "  -q: quiet, suppress some warnings, cancels -v\n"
          "  -v: more verbose (may be repeated), cancels -q\n"
          "  -x: check for # extra articles in each group\n"
          "  -l: do not use supplementary servers\n"
          "  -n: do not automatically expire unread groups\n"
          "  -f: force reload of groupinfo file\n"
          "  -P: only post outgoing articles, don't fetch any\n"
          "  -w: wait, run XOVER updater in foreground\n");
}

/**
 * check whether any of the newsgroups is on server
 * return TRUE if yes, FALSE otherwise
 */
static int
isgrouponserver(const struct server *current_server,
      char *newsgroups /** string will be destroyed! */)
{
    char *p, *q;
    int retval;

    if (!newsgroups)
      return FALSE;

    retval = FALSE;
    p = newsgroups;
    do {
      q = strchr(p, ',');
      if (q)
          *q++ = '\0';
      switch (gs_match(current_server->group_pcre, p)) {
          case 1:
            xsnprintf(lineout, SIZE_lineout, "GROUP %s\r\n", p);
            putaline();
            if (nntpreply(current_server) == 211) {
                if (debug > 1)
                  syslog(LOG_DEBUG, "%s is matched by only_groups_pcre and is on server", p);
                retval = TRUE;
            }
            break;
          case 0:
            if (debug > 1)
                syslog(LOG_DEBUG, "%s not matched by only_group_pcre", p);
            if (current_server->only_groups_match_all) {
                if (debug > 1)
                  syslog(LOG_DEBUG, "not posting article to this server, "
                        "only_groups_match_all is set");
                return FALSE;
            }
            break;
          default:
            break;
      }
      p = q;
      if (p)
          while (*p && isspace((unsigned char)*p))
            p++;
    } while (q);

    return retval;
}

/*
 * check whether message-id is on server
 * return TRUE if yes, FALSE otherwise
 */
static int
ismsgidonserver(const struct server *current_server, char *msgid)
{
    int r;
    if (!msgid)
      return FALSE;
    xsnprintf(lineout, SIZE_lineout, "%s %s\r\n",
            stat_is_evil ? "HEAD" : "STAT", msgid);
    putaline();
    r = nntpreply(current_server);
    if (r >= 220 && r <= 223) {
      if (stat_is_evil)
          ignore_answer(nntpin);
      return TRUE;
    } else
      return FALSE;
}

int
age( /*@null@*/ const char *date)
{
    char monthname[4]; /* RATS: ignore */
    int month;
    int year;
    int day;
    const char *d;
    time_t tmp;
    struct tm time_struct;

    if (!date)
      return 1000;            /* large number: OLD */
    d = date;
    if (!(strncasecmp(d, "date:", 5)))
      d += 5;
    while (isspace((unsigned char)*d))
      d++;

    if (isalpha((unsigned char)*d)) {
      while (*d && !isspace((unsigned char)*d)) /* skip "Mon" or "Tuesday," */
          d++;
    }

    /* RFC 822 says we have 1*LWSP-char between tokens */
    while (isspace((unsigned char)*d))
      d++;

    /* parsing with sscanf leads to crashes */
    day = atoi(d);
    while (isdigit((unsigned char)*d) || isspace((unsigned char)*d))
      d++;
    if (!isalpha((unsigned char)*d)) {
      syslog(LOG_INFO, "Unable to parse %s", date);
      return 1003;
    }
    monthname[0] = *d++;
    monthname[1] = *d++;
    monthname[2] = *d++;
    monthname[3] = '\0';
    if (strlen(monthname) != 3) {
      syslog(LOG_INFO, "Unable to parse month in %s", date);
      return 1004;
    }
    while (isalpha((unsigned char)*d))
      d++;
    while (isspace((unsigned char)*d))
      d++;
    year = atoi(d);

    if ((year < 1970) && (year > 99)) {
      syslog(LOG_INFO, "Unable to parse year in %s", date);
      return 1005;
    } else if (!(day > 0 && day < 32)) {
      syslog(LOG_INFO, "Unable to parse day in %s", date);
      return 1006;
    } else {
      if (!strcasecmp(monthname, "jan"))
          month = 0;
      else if (!strcasecmp(monthname, "feb"))
          month = 1;
      else if (!strcasecmp(monthname, "mar"))
          month = 2;
      else if (!strcasecmp(monthname, "apr"))
          month = 3;
      else if (!strcasecmp(monthname, "may"))
          month = 4;
      else if (!strcasecmp(monthname, "jun"))
          month = 5;
      else if (!strcasecmp(monthname, "jul"))
          month = 6;
      else if (!strcasecmp(monthname, "aug"))
          month = 7;
      else if (!strcasecmp(monthname, "sep"))
          month = 8;
      else if (!strcasecmp(monthname, "oct"))
          month = 9;
      else if (!strcasecmp(monthname, "nov"))
          month = 10;
      else if (!strcasecmp(monthname, "dec"))
          month = 11;
      else {
          syslog(LOG_INFO, "Unable to parse %s", date);
          return 1001;
      }
      if (year < 70)          /* years 2000-2069 in two-digit form */
          year += 100;
      else if (year > 1970)   /* years > 1970 in four-digit form */
          year -= 1900;

      memset(&time_struct, 0, sizeof(time_struct));
      time_struct.tm_sec = 0;
      time_struct.tm_min = 0;
      time_struct.tm_hour = 0;
      time_struct.tm_mday = day;
      time_struct.tm_mon = month;
      time_struct.tm_year = year;
      time_struct.tm_isdst = 0;

      tmp = mktime(&time_struct);

      if (tmp == -1)
          return 1002;

      return ((now - tmp) / SECONDS_PER_DAY);
    }
}

/*
 * Get body of a single message of which the header has already been
 * downloaded and append it to the file with the header.
 * Returns 0 if file could not be retrieved, 1 otherwise.
 */
static int
getbody_insitu(const struct server *current_server, struct newsgroup *group, unsigned long id)
{
    const char *c;
    int rc = 0;
    char *l;
    char *messageid;
    FILE *f;
    char s[SIZE_s+1];
    off_t pos;

    if (!chdirgroup(group->name, FALSE))
      return 0;

    /* extract message-id: header */
    xsnprintf(s, SIZE_s, "%lu", id);
    messageid = getheader(s, "Message-ID:");
    if (!messageid)
      return 0;

    /* check whether we can retrieve the body */
    if (verbose > 2)
      printf("%s: BODY %s\n", group->name, messageid);
    xsnprintf(lineout, SIZE_lineout, "BODY %s\r\n", messageid);
    putaline();

    if (nntpreply(current_server) != 222) {
      ln_log(LNLOG_SERR, LNLOG_CARTICLE,
            "%s: Retrieving body %s failed. No response",
             group->name, messageid);
      rc = 0;
      goto getbody_bail;
    }

    xsnprintf(s, SIZE_s, "%lu", id);
    c = lookup(messageid);
    if (!(f = fopen(c, "a"))) {
      ln_log(LNLOG_SERR, LNLOG_CARTICLE,
            "%s: cannot open %s for appending", group->name, c);
      rc = 0;
      goto getbody_bail;
    }
    pos = ftell(f);
    fputc('\n', f); /* blank line -- separate header and body */

    debug--;
    while (((l = mgetaline(nntpin)) != NULL) && strcmp(l, ".") && !ferror(f)) {
      if (*l == '.')
          ++l;
      fputs(l, f);
      fputc('\n', f);
    }
    debug = debugmode;

    if (!l) {
      ln_log(LNLOG_SERR, LNLOG_CARTICLE, "%s: Retrieving body %s failed. "
             "Transmission interrupted.", group->name, messageid);
      ftruncate(fileno(f), pos);
      fclose(f);
      rc = 0;
      goto getbody_bail;
    }

    /* abort when disk is full */
    if (fclose(f) && errno == ENOSPC) {
      truncate(s, pos);
      raise(SIGINT);
      return 0;
    }

    rc = 1;

  getbody_bail:
    if (messageid)
      free(messageid);
    return rc;
}

static int
getbody_newno(const struct server *current_server, struct newsgroup *group, unsigned long id)
{
    const char *c;
    int rc = 0;
    char *l;
    char *p, *q;
    char *messageid;
    char *newsgroups;         /* I hope this is enough */
    char *xref;
    FILE *f, *g;
    char s[SIZE_s+1];

    if (!chdirgroup(group->name, FALSE))
      return 0;

    /* extract message-id: and xref: headers */
    xsnprintf(s, SIZE_s, "%lu", id);
    if (!(f = fopen(s, "r"))) {
      syslog(LOG_INFO, "%s: cannot open %s for reading -- possibly expired",
             group->name, s);
      return 1;         /* pretend to have read file successfully so that
                           it is purged from the list */
    }
    messageid = NULL;
    newsgroups = NULL;
    xref = NULL;
    debug--;
    while ((l = getaline(f)) != NULL) {
      if (!newsgroups && !strncmp(l, "Newsgroups:", 11)) {
          p = l+11;
          SKIPLWS(p);
          newsgroups = critstrdup(p, "getbody");
      }
      if (!messageid && !strncmp(l, "Message-ID:", 11)) {
          p = l+11;
          SKIPLWS(p);
          messageid = critstrdup(p, "getbody");
      }
      if (!xref && !strncmp(l, "Xref:", 5)) {
          p = l + 5;
          SKIPLWS(p);
          xref = critstrdup(p, "getbody");
      }
    }
    debug = debugmode;
    fclose(f);

    /* check whether we can retrieve the body */
    if (verbose > 2)
      printf("%s: BODY %s\n", group->name, messageid);
    xsnprintf(lineout, SIZE_lineout, "BODY %s\r\n", messageid);
    putaline();

    if (nntpreply(current_server) != 222) {
      ln_log(LNLOG_SERR, LNLOG_CARTICLE, "%s: Retrieving body %s failed. No response",
             group->name, messageid);
      rc = 0;
      goto getbody_bail;
    }

    xsnprintf(s, SIZE_s, "%lu", id);
    c = lookup(messageid);
    log_unlink(c, 0);               /* make space for new file */

    if (!(f = fopen(c, "w"))) {
      ln_log(LNLOG_SERR, LNLOG_CGROUP, "%s: cannot open %s for writing", group->name, c);
      link(s, c);       /* if we can't open new file restore old one */
      rc = 0;
      goto getbody_bail;
    }

    /* copy all headers except Xref: into new file */
    g = fopen(s, "r");
    if (!g) {
      ln_log(LNLOG_SERR, LNLOG_CARTICLE, "%s: open %s failed", group->name, s);
      rc = 0;
      goto getbody_bail;
    }
    debug--;
    while ((l = getaline(g)) != NULL) {
      /* skip xref: headers */
      if (strncasecmp(l, "Xref:", 5) != 0)
          fprintf(f, "%s\n", l);
    }
    debug = debugmode;
    fclose(g);

    /* create a whole bunch of new hardlinks */
    store(c, f, newsgroups, messageid);

    /* retrieve body */
    fprintf(f, "\n");         /* Empty line between header and body. */
    debug--;
    while (((l = mgetaline(nntpin)) != NULL) && strcmp(l, ".") && !ferror(f)) {
      if (*l == '.')
          ++l;
      fputs(l, f);
      fputc('\n', f);
    }
    debug = debugmode;
    if (!l) {
      ln_log(LNLOG_SERR, LNLOG_CARTICLE, "%s: Retrieving body %s failed. "
             "Transmission interrupted.", group->name, messageid);
      fprintf(f, "\n\t[ Leafnode: ]\n"
            "\t[ An error occured while " "retrieving this message. ]\n");
      fclose(f);
      rc = 0;
      goto getbody_bail;
    }
    /* abort when disk is full */
    if (fclose(f) && errno == ENOSPC)
      raise(SIGINT);

    /* Remove old article files since we don't need them anymore.
       This is done by evaluating the old Xref: header.
     */
    if (!xref) {
      /* no Xref: header --> make a fake one */
      xref = critmalloc(50 + strlen(fqdn) + strlen(group->name), "getbody");
      sprintf(xref, "%s %s:%lu", fqdn, group->name, id);    /* RATS: ignore */
    }

    if (debugmode)
      syslog(LOG_DEBUG, "xref: %s", xref);
    c = strchr(xref, ' ');
#ifdef __LCLINT__
    assert(c != NULL);        /* we know c != NULL */
#endif                        /* __LCLINT__ */
    while ((c++) && (*c) && (q = strchr(c, ':')) != NULL) {
      *q++ = '\0';
      if ((p = strchr(q, ' ')) != NULL)
          *p = '\0';

      /* c points to the newsgroup, q to the article number */
      if (!chdirgroup(c, FALSE)) {
          return 0;
      }
      if (unlink(q))
          syslog(LOG_NOTICE,
               "retrieved body, but unlinking headers-only file %s/%s failed",
               c, q);
      else if (debugmode)
          syslog(LOG_DEBUG,
               "retrieved body, now unlinking headers-only file %s/%s", c,
               q);

      c = strchr(q, ' ');
    }
    rc = 1;
  getbody_bail:
    if (xref)
      free(xref);
    if (messageid)
      free(messageid);
    if (newsgroups)
      free(newsgroups);
    return rc;
}

static int
getbody(const struct server *cs, struct newsgroup *group, unsigned long id) {
    static int (*func)(const struct server *, struct newsgroup *, unsigned long);

    if (!func) {
      func = db_situ ? getbody_insitu : getbody_newno;
    }

    return func(cs, group, id);
}

/*
 * Get bodies of messages that have marked for download.
 * The group must already be selected at the remote server and
 * the current directory must be the one of the group.
 */
static void
getmarked(const struct server *current_server, struct newsgroup *group)
{
    int n, i;
    int had_bodies = 0;
    FILE *f;
    mastr *filename = mastr_new(PATH_MAX);
    unsigned long id[BODY_DOWNLOAD_LIMIT]; /* RATS: ignore */
    char *t;

    /* #1 read interesting.groups file */
    n = 0;
    mastr_vcat(filename, spooldir, "/interesting.groups/", group->name, NULL);
    if (!(f = fopen(mastr_str(filename), "r")))
      ln_log(LNLOG_SERR, LNLOG_CGROUP, "Cannot open %s for reading", mastr_str(filename));
    else {
      struct stat st;
      if (fstat(fileno(f), &st) == 0 && st.st_size > 0) {
          had_bodies = 1;
          if (verbose)
            printf("%s: getting bodies of marked messages...\n",
                  group->name);
          while ((t = getaline(f)) && n < BODY_DOWNLOAD_LIMIT) {
            if (sscanf(t, "%lu", &id[n]) == 1)
                ++n;
          }
      }
      fclose(f);
    }
    /* #2 get bodies */
    if (delaybody || had_bodies) {
      syslog(LOG_INFO, "%s: marked bodies %d", group->name, n);
      if (verbose > 1)
          printf("%s: marked bodies %d\n", group->name, n);
    }
    for (i = 0; i < n; ++i)
      if (getbody(current_server, group, id[i]))
          id[i] = 0;

    /* #3 write back ids of all articles which could not be retrieved */
    if (had_bodies) {
      if (!(f = fopen(mastr_str(filename), "w")))
          ln_log(LNLOG_SERR, LNLOG_CGROUP, "Cannot open %s for writing", mastr_str(filename));
      else {
          for (i = 0; i < n; ++i)
            if (id[i] != 0)
                fprintf(f, "%lu\n", id[i]);
          fclose(f);
      }
    }
    if (delaybody || had_bodies) {
      if (verbose)
          printf("%s: Done getting article bodies.\n", group->name);
    }
    mastr_delete(filename);
}

/** count number of colons in the string s_in. */
static int count_colons(const char *s_in) {
    int ngs;
    const char *t;

    ngs = 0;
    t = s_in;
    for(;;) {
      t += strcspn(t, ":");
      if (!*t) break;
      t++;
      ngs++;
      if (ngs < 0) {
          /* overflow */
          return INT_MAX;
      }
    }
    return ngs;
}

/*
 * get newsgroup from a server. "server" is the last article that
 * was previously read from this group on that server
 */
static unsigned long
getgroup(const struct server *current_server,
      /*@null@*/ struct newsgroup *g,
      unsigned long server)
{
#define HD_MAX 10
    static char *hd[HD_MAX];
    const char *hnames[HD_MAX] = { "Path: ", "Message-ID: ", "From: ",
      "Newsgroups: ", "Subject: ", "Date: ",
      "References: ", "Lines: ", "Xref: ", ""
    };

    /* order of headers in XOVER */
    enum enames { /* 0: art. no. */ h_sub = 1, h_fro, h_dat, h_mid,
      h_ref, h_byt, h_lin, h_xref };

    /* XOVER fields: 0 Subject, 1 From, 2 Date, 3 Message-ID, 4
     * References, 5 Bytes, 6 Lines, 7 Xref:full (optional) */

    unsigned long fetched, killed;
    unsigned long h;
    long n;
    unsigned long last;
    unsigned long window;     /* last ARTICLE n command sent */
    char *l;
    FILE *f;
    const char *c;
    struct stat st;
    long outstanding = 0, j;
    unsigned long i;
    unsigned long *stufftoget;
    int localmaxage = maxage;
    int maxagelimit = -1;
    const char *limitfrom = "";
    int expdays;

    if (!g)
      abort();

    if (g->first > g->last && g->first - g->last > 1)
      g->last = g->first - 1;

    if ((expdays = lookup_expiredays(g->name)) > 0) {
      if (localmaxage > expdays) {
          maxagelimit = expdays;
          limitfrom = "groupexpire";
      }
    } else {
      if (localmaxage > expiredays) {
          maxagelimit = expiredays;
          limitfrom = "global expire";
      }
    }

    if (*limitfrom && localmaxage > maxagelimit) {
      if (clamp_maxage) {
          syslog(LOG_NOTICE, "clamping maxage for %s to %s %d",
               g->name, limitfrom, maxagelimit);
          localmaxage = maxagelimit;
      } else {
          fprintf(stderr,
               "warning: group %s: maxage of %d is inappropriate for your "
               "applicable %s of %d. This can cause excessive downloads of "
               "articles that were previously downloaded and expired. "
               "Fix your configuration.\n", g->name, localmaxage, limitfrom,
               maxagelimit);
          syslog(LOG_WARNING,
               "warning: group %s: maxage of %d is inappropriate for your "
               "applicable %s of %d. This can cause excessive downloads of "
               "articles that were previously downloaded and expired. "
               "Fix your configuration.", g->name, localmaxage, limitfrom,
               maxagelimit);
      }
    }

    if (server == 0ul)
      server = 1ul;

    /* skip */
    if (gs_match(current_server->group_pcre, g->name) != 1) {
      if (verbose > 1) {
          printf("%s: skipped %s, not in only_groups_pcre\n",
                current_server->name, g->name);
      }
      return server;
    }

    /* skip */
    if ((l = getenv("LN_SKIP_GROUPS"))) {
      char *x, *y = critstrdup(l, "getgroup");
      for (x = strtok(y, ","); x; x = strtok(NULL, ",")) {
          if (wildmat(g->name, x)) {
            if (verbose) {
                printf("%s: skipped %s, LN_SKIP_GROUPS=%s\n",
                      current_server->name, g->name, l);
                syslog(LOG_INFO, "%s: skipped %s, LN_SKIP_GROUPS=%s",
                      current_server->name, g->name, l);
            }
            free(y);
            return server;
          }
      }
      free(y);
    }

    xsnprintf(lineout, SIZE_lineout, "GROUP %s\r\n", g->name);
    putaline();

    n = nntpreply(current_server);
    if (n == 498) {
      return 0;
    }
    l = lastreply();

    if (n == 411) {           /* group not available on server */
      if (verbose > 1)
          printf("%s: no such group\n", g->name);
      return server;
    }

    if (sscanf(l, "%3ld %lu %lu %lu ", &n, &h, &window, &last) < 4 || n != 211)
    {
      fprintf(stderr, "Warning: %s: cannot parse server reply \"%s\"\n", g->name, l);
      syslog(LOG_WARNING, "Warning: %s: cannot parse server reply \"%s\"", g->name, l);
      return 0;
    }

    if (h == 0) {
      if (verbose > 1)
          printf("%s: upstream group is empty\n", g->name);
      return server;
    }

    if (extraarticles) {
      if (server > extraarticles)
          i = server - extraarticles;
      else
          i = 1;
      if (i < window)
          i = window;
      if (verbose > 1 && server > 1)
          printf("%s: backing up from %lu to %lu\n", g->name, server, i);
      server = i;
    }

    if (server > last + 1) {
      syslog(LOG_INFO,
             "%s: last seen article was %lu, server now has %lu-%lu",
             g->name, server, window, last);
      if (server > last + 5) {
          if (verbose)
            printf("%s: switched upstream servers? %lu > %lu\n",
                   g->name, server - 1, last);
          server = window;    /* insane - recover thoroughly */
      } else {
          if (verbose)
            printf("%s: rampant spam cancel? %lu > %lu\n", g->name, server - 1, last);
          server = last - 5;  /* a little bit too much */
      }
    }

    if (initiallimit && server == 1 && last > server
      && last - server > initiallimit) {
      if (verbose > 1)
          printf("%s: skipping articles %lu-%lu inclusive (initial limit)\n",
               g->name, server, last - initiallimit);
      syslog(LOG_INFO, "%s: skipping articles %lu-%lu inclusive (initial limit)",
             g->name, server, last - initiallimit);
      server = last - initiallimit + 1;
    }

    if (artlimit && last > server && last - server > artlimit) {
      if (verbose > 1)
          printf("%s: skipping articles %lu-%lu inclusive (article limit)\n",
               g->name, server, last - artlimit - 1);
      syslog(LOG_INFO, "%s: skipping articles %lu-%lu inclusive (article limit)",
             g->name, server, last - artlimit - 1);
      server = last - artlimit;
    }

    getmarked(current_server, g);

    if (window < server)
      window = server;
    if (window < 1)
      window = 1;
    server = window;

    if (server > last) {
      if (verbose > 1)
          printf("%s: no new articles\n", g->name);
      syslog(LOG_INFO, "%s: no new articles\n", g->name);
      return server;
    }

    if (verbose > 1)
      printf("%s: considering articles %lu - %lu\n", g->name, server, last);
    syslog(LOG_INFO, "%s: considering articles %lu - %lu\n", g->name, server,
         last);

    fetched = 0;
    killed = 0;

    stufftoget =
      (unsigned long *)malloc(sizeof(stufftoget[0]) * (last + 1 - server));
    if (!stufftoget) {
      ln_log(LNLOG_SERR, LNLOG_CGROUP, "not enough memory for XHDRs");
      return server;
    }
    memset(stufftoget, 0, sizeof(stufftoget[0]) * (last + 1 - server));

    if (!current_server->noxover) {
      int tmp;

      /* Try XOVER */
      xsnprintf(lineout, SIZE_lineout, "XOVER %lu-%lu\r\n", server, last);
      putaline();
      if (nntpreply(current_server) == 224) {
          mastr *ol = mastr_new(1024);

          debug--;
          while ((l = mgetaline(nntpin)) && strcmp(l, ".")) {
            /*@dependent@*/ char *fields[HD_MAX];
            unsigned long art;
            const char *t;
            char *q;

            mastr_cpy(ol, l);
            /* split xover */
            /*@+loopexec@*/
            for (i = 0; l && l[0] && i < HD_MAX; i++) {
                char *y;
                fields[i] = l;
                if ((y = strchr(l, '\t'))) {
                  y[0] = '\0';
                  l = y + 1;
                } else {
                  l = NULL;
                };
            };
            /*@=loopexec@*/
            /* short line -- log and skip */
            if (i < 7) {
                ln_log(LNLOG_SWARNING, LNLOG_CTOP,
                      "%s: %s: Warning: got unparsable XOVER line from server, "
                      "too few fields (%lu): \"%s\"",
                      current_server->name, g->name, i, mastr_str(ol));
                continue;
            }
            for (; i < HD_MAX; i++)
                fields[i] = NULL;
            art = strtoul(fields[0], &q, 10);
            if (q && art >= server && art <= last) {
                long artlines; /* invalid: -1 */
                unsigned long artbytes; /* invalid: 0 */
                if (fields[h_lin]) artlines = strtol(fields[h_lin], NULL, 10);
                else artlines = -1;
                if (fields[h_byt]) artbytes = strtoul(fields[h_byt], NULL, 10);
                else artbytes = 0;

                if (maxbytes && fields[h_byt]
                      && (strtoul(fields[h_byt], NULL, 10) > maxbytes)) {
                  killed++;
                  ln_log(LNLOG_SINFO, 4, "%s: killed %lu (%s), "
                        "too many bytes (%lu > %lu)",
                        g->name, art, fields[h_mid],
                        strtoul(fields[h_byt], NULL, 10), maxbytes);
                } else if (maxbytes && artbytes != 0
                      && artbytes > maxbytes) {
                  killed++;
                  ln_log(LNLOG_SINFO, 4, "%s: killed %lu (%s), "
                        "too large (%lu > %lu)",
                        g->name, art, fields[h_mid],
                        artbytes, maxbytes);
                } else if (maxlines && artlines != -1
                      && artlines > maxlines) {
                  killed++;
                  ln_log(LNLOG_SINFO, 4, "%s: killed %lu (%s), "
                        "too many lines (%ld > %ld)",
                        g->name, art, fields[h_mid],
                        artlines, maxlines);
                } else if (minlines && artlines != -1
                      && artlines < minlines) {
                  killed++;
                  ln_log(LNLOG_SINFO, 4, "%s: killed %lu (%s), "
                        "too few lines (%ld < %ld)",
                        g->name, art, fields[h_mid],
                        artlines, minlines);
                } else if (localmaxage && fields[h_dat]
                      && (age(fields[h_dat]) > localmaxage)) {
                  killed++;
                  ln_log(LNLOG_SINFO, 4, "%s: killed %lu (%s), "
                        "too old (%d > %d) days",
                        g->name, art, fields[h_mid],
                        age(fields[h_dat]), localmaxage);
                } else if (crosspostlimit && fields[h_xref] && (tmp = count_colons(fields[h_xref]) - 1) > crosspostlimit) {
                  /* -1 to skip over the header's name, "Xref:" */
                  killed++;
                  ln_log(LNLOG_SINFO, 4, "%s: killed %lu (%s), "
                        "too many groups in Xref: header (%d > %ld)",
                        g->name, art, fields[h_mid],
                        tmp, crosspostlimit);
                } else if (lstat(t = lookup(fields[h_mid]), &st) == 0) {
                  killed++;
                  ln_log(LNLOG_SINFO, 4, "%s: killed %lu (%s), already fetched before",
                        g->name, art, fields[h_mid]);
                } else {
                  stufftoget[outstanding] = art;
                  outstanding++;
                  if (verbose > 2)
                      printf("%s: will fetch %lu (%s)\n", g->name, art, t);
                }
            }
          }
          mastr_delete(ol);
          debug = debugmode;
          goto have_outstanding;
      }

      ln_log(LNLOG_SINFO, 2, "XOVER failed, trying XHDR");
    }

    xsnprintf(lineout, SIZE_lineout, "XHDR Message-ID %lu-%lu\r\n", server,
            last);
    putaline();
    if (nntpreply(current_server) == 221) {
      debug--;
      while ((l = mgetaline(nntpin)) && strcmp(l, ".")) {
          unsigned long art;
          char *t;
          art = strtoul(l, &t, 10);
          if (t && isspace((unsigned char)*t)) {
            while (isspace((unsigned char)*t))
                t++;
            if (art >= server && art <= last && stat(lookup(t), &st) != 0) {
                stufftoget[outstanding] = art;
                outstanding++;
                if (verbose > 2)
                  printf("%s: will fetch %lu (%s)\n", g->name, art, t);
            }
          }
      }
      debug = debugmode;
      if (!l) {
          free(stufftoget);
          ln_log(LNLOG_SWARNING, LNLOG_CARTICLE,
                "warning: %s: %s: server disconnect or timeout after XHDR",
                current_server->name, g->name);
          return 0;
      }
    } else {
      free(stufftoget);
      return server;
    }

    if (outstanding == 0) {
      free(stufftoget);
      return last + 1;
    }

    syslog(LOG_INFO, "%s: will fetch %ld article%s", g->name, outstanding, PLURAL(outstanding));
    if (verbose > 1)
      printf("%s: will fetch %ld article%s\n", g->name, outstanding, PLURAL(outstanding));

    if (minlines || maxlines) {
      xsnprintf(lineout, SIZE_lineout, "XHDR Lines %lu-%lu\r\n", server,
              last);
      putaline();
      if (nntpreply(current_server) == 221) {
          while ((l = mgetaline(nntpin)) && strcmp(l, ".")) {
            unsigned long art;
            long lines = 0;
            char *t;
            art = strtoul(l, &t, 10);
            if (t)
                lines = strtol(t, NULL, 10);
            for (j = 0; j < outstanding; j++) {
                if (art == stufftoget[j])
                  break;
            }
            if (j < outstanding)
                if ((minlines && lines < minlines)
                      || (maxlines && lines > maxlines)) {
                stufftoget[j] = 0;
                syslog(LOG_INFO, "%s: Killed article %lu: %ld line%s",
                     g->name, art, lines, PLURAL(lines));
                if (verbose > 2)
                  printf("%s: Killed article %lu: %ld line%s.\n",
                         g->name, art, lines, PLURAL(lines));
                killed++;
            }
          }
          if (!l) {           /* timeout */
            free(stufftoget);
            ln_log(LNLOG_SWARNING, LNLOG_CARTICLE,
                  "warning: %s: %s: server disconnect or timeout after XHDR Lines",
                  current_server->name, g->name);
            return 0;
          }
      }
    }

    if (maxbytes) {
      xsnprintf(lineout, SIZE_lineout, "XHDR Bytes %lu-%lu\r\n", server,
              last);
      putaline();
      if (nntpreply(current_server) == 221) {
          while ((l = mgetaline(nntpin)) && strcmp(l, ".")) {
            unsigned long art, bytes = 0;
            char *t;
            art = strtoul(l, &t, 10);
            if (t)
                bytes = strtoul(t, NULL, 10);
            for (j = 0; j < outstanding; j++) {
                if (art == stufftoget[j])
                  break;
            }
            if (j < outstanding && (bytes > maxbytes)) {
                stufftoget[j] = 0;
                syslog(LOG_INFO, "%s: Killed article %lu (%lu > %lu bytes)",
                     g->name, art, bytes, maxbytes);
                if (verbose > 2)
                  printf("%s: Killed article %lu (%lu > %lu bytes).\n",
                         g->name, art, bytes, maxbytes);
                killed++;
            }
          }
          if (!l) {           /* timeout */
            free(stufftoget);
            ln_log(LNLOG_SWARNING, LNLOG_CARTICLE,
                  "warning: %s: %s: server disconnect or timeout after XHDR Bytes",
                  current_server->name, g->name);

            return 0;
          }
      }
    }

    if (localmaxage) {
      xsnprintf(lineout, SIZE_lineout, "XHDR Date %lu-%lu\r\n", server, last);
      putaline();
      if (nntpreply(current_server) == 221) {
          debug--;
          while ((l = mgetaline(nntpin)) && strcmp(l, ".")) {
            unsigned long art;
            long aage = 0;
            char *t;

            art = strtoul(l, &t, 10);
            if (t)
                aage = age(t);
            for (j = 0; j < outstanding; j++) {
                if (art == stufftoget[j])
                  break;
            }
            if (j < outstanding && (aage > localmaxage)) {
                stufftoget[j] = 0;
                syslog(LOG_INFO,
                     "%s: Killed article %lu (%ld > %d = localmaxage)", g->name,
                     art, aage, localmaxage);
                if (verbose > 2)
                  printf("%s: Killed article %lu (%ld > %d = localmaxage).\n",
                         g->name, art, aage, localmaxage);
                killed++;
            }
          }
      
          if (!l) {           /* timeout */
            free(stufftoget);
            ln_log(LNLOG_SWARNING, LNLOG_CARTICLE,
                  "warning: %s: %s: server disconnect or timeout after XHDR Date",
                  current_server->name, g->name);

            return 0;
          }

          debug = debugmode;
      }
    }

    /* now we have a list of articles in stufftoget[] */
    /* let's get the header and possibly bodies of these */
  have_outstanding:
    for (i = 0; outstanding > 0; i++) {
      int takethis = 1;
      int requested_body;
      const char *cmd;

      outstanding--;
      if (!stufftoget[i])
          continue;

      if (stufftoget[i] < server) {
          if (verbose > 2)
            printf("%s: skipping %lu - not available or too old\n",
                   g->name, stufftoget[i]);
          syslog(LOG_INFO, "%s: skipping %lu - not available or too old",
               g->name, stufftoget[i]);
          continue;
      }

      debug = debugmode;
      requested_body = ((!filterfile || article_despite_filter) && !delaybody);
      cmd = requested_body ? "ARTICLE" : "HEAD";
      xsnprintf(lineout, SIZE_lineout, "%s %lu\r\n", cmd, stufftoget[i]);
      putaline();
      l = mgetaline(nntpin);
      /* timeout */
      if (!l)
      {
          ln_log(LNLOG_SERR, LNLOG_CARTICLE,
                "Server disconnection or timeout before article "
                "could be retrieved #1");
          free(stufftoget);
          return 0;
      }
      /* check proper reply code */
      if (sscanf(l, "%3ld %lu", &n, &h) < 2 || ((n / 10) != 22)) {
          if (verbose > 2)
            printf("%s %s %lu: reply %s (%ld more up in the air)\n",
                   g->name, cmd, stufftoget[i], l, outstanding);
          syslog(LOG_INFO, "%s %s %lu: reply %s (%ld more up in the air)",
               g->name, cmd, stufftoget[i], l, outstanding);
          continue;
      }

      /* anything below this line will have to make sure that data is
       * drained properly in case */

      debug--;
      if (verbose > 2)
          printf("%s: receiving article %lu (%ld more up in the air)\n",
               g->name, stufftoget[i], outstanding);

      for (h = 0; h < 10; h++) {
          if (hd[h])
            free(hd[h]);
          hd[h] = critstrdup("", "getgroup");
      }
      c = NULL;
      n = 9;                  /* "other" header */
      while ((l = getfoldedline(nntpin)) && *l && strcmp(l, ".")) {
          /* regexp pattern matching */
          if (filterfile && dofilter(l)) {
            killed++;
            if (verbose > 2)
                printf(".filtered article %lu: match on \"%s\"\n",
                      stufftoget[i], l);
            syslog(LOG_INFO, "filtered article %lu: match on \"%s\"",
                   stufftoget[i], l);
            takethis = 0;
            free(l);
            l = NULL;
            continue;
          }

          n = 0;
          while (strncasecmp(l, hnames[n], strlen(hnames[n])))
            n++;
          if (n < 9 && hd[n] && *(hd[n]))
            /* second occurance of the same recognized header
             * is treated as if it was not listed
             * in hnames (as "other header") */
            n = 9;
          hd[n] = critrealloc(hd[n], strlen(hd[n]) + strlen(l) + 2,
                        "Fetching article header");
          if (strlen(hd[n]))
            strcat(hd[n], "\n");    /* RATS: ignore */
          strcat(hd[n], l);   /* RATS: ignore */
          if (debugmode > 1 && verbose > 3 && hnames[n] && *hnames[n])
            printf("...saw header %s\n", hnames[n]);
          free(l);
          l = NULL;
      } /* end while */

      /* if server ended the fetch prematurely, assume we didn't
       * request a body to ignore - ignoring would cause timeout
       * waiting for data that is never sent.
       *
       * SourceForge bug 873149, reported 2004-01-08 by Toni Viemerö,
       * sourceforge user "skithund" */
      if (l == NULL) {
          /* timeout - don't flush body */
          requested_body = FALSE;
      } else if (strcmp(l, ".") == 0 && requested_body) {
          ln_log(LNLOG_SNOTICE, LNLOG_CTOP, "%s: %s:%lu: article without blank line after header, format violation",
                current_server->name, g->name, stufftoget[i]);
          requested_body = FALSE;
      }

      if (l)
          free(l);

      if (!takethis) {
          if (requested_body) ignore_answer(nntpin);
          continue;           /* filtered article */
      }

      debug = debugmode;
      if (!l) {         /* timeout */
          free(stufftoget);
          ln_log(LNLOG_SERR, LNLOG_CARTICLE,
                "Server disconnection or timeout before article "
                "could be retrieved #2");
          return 0;
      }

      /* check headers */
      for (h = 0; h < 6; h++) {
          if (!hd[h] || !*(hd[h])) {
            if (verbose)
                printf("Discarding article %lu - no %s found\n",
                     stufftoget[i], hnames[h]);
            syslog(LOG_NOTICE,
                   "Discarding article %lu - no %s found",
                   stufftoget[i], hnames[h]);
            killed++;
            if (requested_body) ignore_answer(nntpin);
            takethis = 0;
            break;
          }
      }

      /* mandatory header missing */
      if (!takethis)
          continue;

      if (localmaxage && age(hd[5]) > localmaxage) {
          if (verbose > 2)
            printf("Discarding article %lu - older than %d day%s\n",
                   stufftoget[i], localmaxage, PLURAL(localmaxage));
          syslog(LOG_INFO, "Discarding article %lu %s - older than %d day%s",
               stufftoget[i], hd[4], localmaxage, PLURAL(localmaxage));
          killed++;
          if (requested_body) ignore_answer(nntpin);
          continue;
      }

      if (minlines || maxlines) {
          char *t;
          t = strchr(hd[7], ' ');
          if (t) {
            n = strtol(t, NULL, 10);
            if (minlines && n < minlines) {
                if (verbose > 2)
                  printf("Discarding article %lu - %ld < minlines\n",
                         stufftoget[i], n);
                syslog(LOG_INFO,
                     "Discarding article %lu %s -- %ld < minlines",
                     stufftoget[i], hd[4], n);
                killed++;
                if (requested_body) ignore_answer(nntpin);
                continue;
            }
            if (maxlines && n > maxlines) {
                if (verbose > 2)
                  printf("Discarding article %lu - %ld > maxlines\n",
                         stufftoget[i], n);
                syslog(LOG_INFO,
                     "Discarding article %lu %s -- %ld > maxlines",
                     stufftoget[i], hd[4], n);
                killed++;
                if (requested_body) ignore_answer(nntpin);
                continue;
            }
          }
      }

      if (crosspostlimit) {
          char *t;
          t = hd[3];
          n = 1;        /* number of groups the article is posted to */
          while ((t = strchr(t, ',')) != NULL) {
            t++;
            n++;
          }
          if (crosspostlimit < n) {
            if (verbose > 2)
                printf("Discarding article %lu - posted to %ld groups "
                     "(max. %ld)\n", stufftoget[i], n, crosspostlimit);
            syslog(LOG_INFO,
                   "Discarding article %lu %s - posted to %ld groups "
                   "(max. %ld)", stufftoget[i], hd[4], n, crosspostlimit);
            killed++;
            if (requested_body) ignore_answer(nntpin);
            continue;
          }
      }

      /* store articles */
      f = NULL;
      c = lookup(strchr(hd[1], '<'));     /* lookup also replaces '/' with '@' */

      if (!c) {
          ln_log(LNLOG_SERR, LNLOG_CARTICLE, "lookup of %s failed", hd[1]);
          if (requested_body) ignore_answer(nntpin);
          continue;
      }

      if (!stat(c, &st)) {
          syslog(LOG_INFO, "article %s already stored", c);
          if (requested_body) ignore_answer(nntpin);
          continue;           /* for some reasons, article is already there */
      } else if (errno == ENOENT) {
          f = fopen(c, "w");
          if (!f) {
            ln_log(LNLOG_SERR, LNLOG_CARTICLE, "unable to create article %s: %m", c);
            if (requested_body) ignore_answer(nntpin);
            continue;
          }
      } else {
          ln_log(LNLOG_SERR, LNLOG_CARTICLE, "unable to store article %s: %m", c);
          if (requested_body) ignore_answer(nntpin);
          continue;
      }

      for (h = 0; h < 10; h++)
          if (h != 8 && hd[h] && *(hd[h]))
            fprintf(f, "%s\n", hd[h]);

      h = 0;
      /* replace tabs and other odd signs with spaces */
      while (h < 8) {
          char *p1;
          char *p2;
          p1 = p2 = hd[h];
          while (p1 && *p1) {
            if (isspace((unsigned char)*p1)) {
                *p2 = ' ';
                do {
                  p1++;
                } while (isspace((unsigned char)*p1));
            } else {
                *p2 = *p1++;
            }
            p2++;
          }
          *p2 = '\0';
          h++;
      }

      if (fflush(f)) {
          (void)fclose(f);
          (void)unlink(c);
          if (requested_body) ignore_answer(nntpin);
          continue;
      }

      /* generate hardlinks; this procedure also increments g->last */
      store(c, f, *hd[3] ? hd[3] + strlen(hnames[3]) : "",
            *hd[1] ? hd[1] + strlen(hnames[1]) : "");

      if (delaybody) {
          if (fclose(f)) {
            int e = errno;
            (void)truncate(c, 0);
            (void)unlink(c);
            if (e == ENOSPC)
                raise(SIGINT);
          } else {
            fetched++;
          }
          continue;
      }

      if (!requested_body) {
          xsnprintf(lineout, SIZE_lineout, "BODY %lu\r\n", stufftoget[i]);
          putaline();
          l = mgetaline(nntpin);
          if (!l) {           /* timeout */
            (void)fflush(f);
            (void)ftruncate(fileno(f), 0);
            (void)fclose(f);
            unlink(c);
            ln_log(LNLOG_SWARNING, LNLOG_CARTICLE,
                  "warning: %s: %s: server disconnect or timeout after BODY %lu",
                  current_server->name, g->name, stufftoget[i]);
            free(stufftoget);
            return 0;
          }
          if (sscanf(l, "%3ld", &n) != 1 || (n / 10 != 22)) {
            if (verbose > 2)
                printf("BODY %lu: reply %s\n", stufftoget[i], l);
            syslog(LOG_NOTICE, "BODY %lu: reply %s", stufftoget[i], l);
            (void)fflush(f);
            (void)ftruncate(fileno(f), 0);
            (void)fclose(f);
            unlink(c);
            continue;
          }
      }
      debug--;
      fputs("\n", f);         /* empty line between header and body */
      while (((l = mgetaline(nntpin)) != NULL) && strcmp(l, ".")) {
          if (*l == '.')
            l++;
          clearerr(f);
          fputs(l, f);
          fputc('\n', f);
          if (feof(f)) {
            l = NULL;
            break;
          }
      }
      debug = debugmode;
      fetched++;
      if (fflush(f)) {
          l = NULL;
      }
      if (fclose(f)) {
          l = NULL;
      }
      if (l == NULL) {  /* article didn't terminate with a .: error */
          (void)truncate(c, 0);
          (void)unlink(c);
            ln_log(LNLOG_SWARNING, LNLOG_CARTICLE,
                  "warning: %s: %s: server disconnect or timeout retrieving article %lu",
                  current_server->name, g->name, stufftoget[i]);
          free(stufftoget);
          return 0;
      }
    }

    syslog(LOG_INFO, "%s: %lu article%s fetched (to %lu), %lu killed",
         g->name, fetched, PLURAL(fetched), g->last, killed);
    if (verbose > 1)
      printf("%s: %lu article%s fetched, %lu killed\n",
             g->name, fetched, PLURAL(fetched), killed);
    free(stufftoget);
    return last + 1;
}

/* return 1 == success, 0 == failure */
static int expire_interesting(void) {
    DIR *d;
    struct dirent *de;
    char s[SIZE_s+1];

    xsnprintf(s, SIZE_s, "%s/interesting.groups/", spooldir);

    d = opendir(s);
    if (d == NULL) {
      ln_log(LNLOG_SERR, LNLOG_CTOP, "cannot open %s for reading: %m", s);
      return 0;
    }

    while ((de = readdir(d))) {
      struct stat st;

      xsnprintf(s, SIZE_s, "%s/interesting.groups/%s", spooldir, de->d_name);
      if (stat(s, &st) < 0)
          continue;
      /* reading a newsgroup changes the ctime; if the newsgroup is
         newly created, the mtime is changed as well */
      if (((st.st_mtime == st.st_ctime) &&
           (now - st.st_ctime > (timeout_short * SECONDS_PER_DAY))) ||
          (now - st.st_ctime > (timeout_long * SECONDS_PER_DAY))) {
          if (verbose > 1)
            printf("unsubscribing from %s\n", de->d_name);
          syslog(LOG_INFO, "unsubscribing from %s (current time: %ld): "
               "ctime age %ld, mtime age %ld", de->d_name, (long)now,
               (long)now - st.st_ctime, (long)now - st.st_mtime);
          unlink(s);
      }
    }

    (void)closedir(d);
    return 1;
}

static void formatserver(char *d, size_t len, const struct server *serv, const char *suffix)
{
    if (serv->port == 0 || serv->port == 119)
      xsnprintf(d, len, "%s/leaf.node/%s%s", spooldir, serv->name, suffix);
    else
      xsnprintf(d, len, "%s/leaf.node/%s:%u%s", spooldir, serv->name,
            serv->port, suffix);
}

/** get active file from current_server.
 * \returns 0 for success, non-zero for error.
 */
static int
nntpactive(struct server *current_server, time_t *stamp)
{
    struct stat st;
    char *l, *p;
    struct stringlist *groups = NULL;
    struct stringlist *helpptr = NULL;
    char timestr[64]; /* RATS: ignore */
    long reply = 0l;
    int error, merge = 0;
    char s[SIZE_s+1];
    int try_xgtitle = 1;
    static time_t cur_date;
    static int cur_date_init = 0;
    time_t t;

    if (!cur_date_init) {
      cur_date = time(NULL);
      cur_date_init = 1;
    }

    formatserver(s, SIZE_s, current_server, "");
    t = time(NULL);
    if (active && !forceactive && (stat(s, &st) == 0)) {
      if (verbose)
          printf("%s: getting new newsgroups\n", current_server->name);
      /* to avoid a compiler warning we print out a four-digit year;
       * but since we need only the last two digits, we skip them
       * in the next line
       */
      *stamp = st.st_mtime;
      (void)strftime(timestr, sizeof(timestr),
                   "%Y%m%d %H%M%S", gmtime(&st.st_mtime));
      xsnprintf(lineout, SIZE_lineout, "NEWGROUPS %s GMT\r\n", timestr + 2);
      putaline();
      /* we used to expect 231 here, but some broken servers (MC-link
       * Custom News-server V1.06) return 215 instead.
       * Just accept any 2XX code as success.
       */
      if ((reply = nntpreply(current_server)) < 200 || reply >= 300) {
          char *e = lastreply();
          if (!e) e = "server disconnect or timeout";
          ln_log(LNLOG_SERR, LNLOG_CSERVER,
                "%s: reading new newsgroups failed, reason \"%s\"",
                current_server->name, e);
          return -1;
      }
      while ((l = mgetaline(nntpin)) && (strcmp (l, "."))) {
          p = l;
          while (*p && !isspace((unsigned char)*p))
            p++;
          if (*p)
            *p = '\0';
          if (gs_match(current_server->group_pcre, l)) {
            merge++;
            insertgroup(l, 1, 0, cur_date);
            prependtolist(&groups, l);
          }
      }
      if (!l) {         /* timeout */
          ln_log(LNLOG_SERR, LNLOG_CSERVER,
                "%s: reading new newsgroups failed, server disconnect or timeout.",
                current_server->name);
          return -1;
      }
      ln_log(LNLOG_SINFO, LNLOG_CSERVER,
                "%s: got %d new newsgroups.",
                current_server->name, merge);
      if (merge) {
          mergegroups();      /* merge groups into active */
          merge = 0;
      }
      helpptr = groups;
      if (verbose && helpptr && current_server->descriptions)
          printf("%s: getting newsgroup descriptions\n",
                current_server->name);
      while (helpptr != NULL) {
          if (current_server->descriptions) {
            error = 0;
            if (try_xgtitle) {
                xsnprintf(lineout, SIZE_lineout, "XGTITLE %s\r\n",
                      helpptr->string);
                putaline();
                reply = nntpreply(current_server);
            }
            if (!try_xgtitle || reply != 282) {
                try_xgtitle = 0;
                xsnprintf(lineout, SIZE_lineout, "LIST NEWSGROUPS %s\r\n",
                        helpptr->string);
                putaline();
                reply = nntpreply(current_server);
                if (reply && (reply != 215))
                  error = 1;
            }
            if (!error) {
                l = mgetaline(nntpin);
                if (l && *l && strcmp(l, ".")) {
                  p = l;
                  while (*p && !isspace((unsigned char)*p))
                      p++;
                  while (isspace((unsigned char)*p)) {
                      *p = '\0';
                      p++;
                  }
                  if (reply == 215 || reply == 282)
                      changegroupdesc(l, *p ? p : NULL);
                  do {
                      l = mgetaline(nntpin);
                      error++;
                  } while (l && *l && strcmp(l, "."));
                  if (error > 1) {
                      current_server->descriptions = 0;
                      syslog(LOG_WARNING, "warning: %s does not process "
                           "LIST NEWSGROUPS %s correctly: use nodesc\n",
                           current_server->name, helpptr->string);
                      fprintf(stderr, "warning: %s does not process LIST "
                           "NEWSGROUPS %s correctly: use nodesc\n",
                           current_server->name, helpptr->string);
                  }
                }
            }
          }             /* if ( current_server->descriptions ) */
          helpptr = helpptr->next;
      }
      freelist(groups);
    } else {
      ln_log(LNLOG_SINFO, LNLOG_CSERVER,
          "%s: getting all newsgroups (debug: active: %s, forceactive: %s)",
            current_server->name,
            active ? "set" : "nil", forceactive ? "true" : "false");
      xsnprintf(lineout, SIZE_lineout, "LIST\r\n");
      putaline();
      if (nntpreply(current_server) != 215) {
          char *e = lastreply();
          if (!e) e = "server disconnect or timeout";
          ln_log(LNLOG_SERR, LNLOG_CSERVER,
                "%s: reading all newsgroups failed, reason \"%s\".",
                current_server->name, e);
          return -2;
      }
      debug--;
      while ((l = mgetaline(nntpin)) && (strcmp(l, "."))) {
          p = l;
          while (*p && !isspace((unsigned char)*p))
            p++;
          while (isspace((unsigned char)*p)) {
            *p = '\0';
            p++;
          }
          if (gs_match(current_server->group_pcre, l)) {
            insertgroup(l, 1, 0, cur_date);
          }
      }
      mergegroups();
      if (!l) {         /* timeout */
          ln_log(LNLOG_SERR, LNLOG_CSERVER,
                "%s: reading all newsgroups failed, server disconnect or timeout.",
                current_server->name);
          return -2;
      }
      if (current_server->descriptions) {
          if (verbose)
            printf("%s: getting newsgroup descriptions\n",
                  current_server->name);
          xsnprintf(lineout, SIZE_lineout, "LIST NEWSGROUPS\r\n");
          putaline();
          l = mgetaline(nntpin);
          /* correct reply starts with "215". However, INN 1.5.1 is broken
             and immediately returns the list of groups */
          if (l) {
            if (debug)
                syslog(LOG_DEBUG, "<%s", l);
            reply = strtol(l, &p, 10);
            if ((reply == 215) && (*p == ' ' || *p == '\0')) {
                l = mgetaline(nntpin);    /* get first description */
            } else if (*p != ' ' && *p != '\0') {
                int dummy = 0;
                /* INN 1.5.1: line already contains description */
                (void)dummy;
            } else {
                ln_log(LNLOG_SERR, LNLOG_CSERVER,
                      "%s: reading newsgroups descriptions failed: %s",
                      current_server->name, l);
                ln_log(LNLOG_SERR, LNLOG_CSERVER,
                      "Workaround: Add \"nodesc = 1\" (without quotes) below the server = %s line.", current_server->name);
                return -2;
            }
          } else {
            ln_log(LNLOG_SERR, LNLOG_CSERVER, 
                  "%s: reading newsgroups descriptions failed: server disconnect or timeout.",
                  current_server->name);
            return -2;
          }
          while (l && (strcmp(l, "."))) {
            p = l;
            while (*p && !isspace((unsigned char)*p))
                p++;
            while (isspace((unsigned char)*p)) {
                *p = '\0';
                p++;
            }
            changegroupdesc(l, *p ? p : NULL);
            l = mgetaline(nntpin);
          }
          if (!l) {           /* timeout */
            ln_log(LNLOG_SERR, LNLOG_CSERVER,
                  "%s: reading newsgroup descriptions failed, server disconnect or timeout.",
                  current_server->name);
            return -2;
          }
      }
      debug = debugmode;

      /* mark active for /this/ server fetched */
      {
          FILE *f = fopen(s, "a");
          if (!f) {
            ln_log(LNLOG_SERR, LNLOG_CGROUP,
                  "cannot open \"%s\": %m", s);
          } else {
            if (fclose(f))
                ln_log(LNLOG_SERR, LNLOG_CGROUP,
                      "cannot close \"%s\": %m", s);
          }
      }
    }
    *stamp = t;
    return 0;
}

static int
movetofailed(const char *name) {
    char s[SIZE_s + 1];

    xsnprintf(s, SIZE_s, "%s/failed.postings/%s", spooldir, name);
    ln_log(LNLOG_SERR, LNLOG_CARTICLE,
          "moving file %s to failed.postings", name);
    if (rename(name, s)) {
      ln_log(LNLOG_SERR, LNLOG_CARTICLE,
            "unable to move failed posting to %s: %m", s);
      return -1;
    } else {
      return 0;
    }
}

/*
 * post all spooled articles
 *
 * if all postings succeed, returns 1
 * if there are no postings to post, returns 1
 * if a posting is strange for some reason, returns 0
 */
static int
postarticles(const struct server *current_server)
{
    struct stat st;
    char *line;
    DIR *d;
    struct dirent *de;
    FILE *f;
    int r, haveid, n;
    char *p, *q;
    int savedir;

    n = 0;

    savedir = open(".", O_RDONLY);
    if (savedir < 0) {
      ln_log(LNLOG_SERR, LNLOG_CTOP,
            "postarticles: Unable to save current working directory: %m");
      return 0;
    }

    if (chdir(spooldir) || chdir("out.going")) {
      ln_log(LNLOG_SERR, LNLOG_CTOP,
            "postarticles: Unable to cd to %s/out.going: %m",
            spooldir);
      fchdir(savedir);
      close(savedir);
      return 0;
    }

    d = opendir(".");
    if (!d) {
      ln_log(LNLOG_SERR, LNLOG_CTOP,
            "postarticles: Unable to opendir %s/out.going: %m", spooldir);
      fchdir(savedir);
      close(savedir);
      return 0;
    }

    while ((de = readdir(d)) != NULL) {
      haveid = 0;
      f = NULL;
      if (!strcmp(".", de->d_name) || !strcmp("..", de->d_name)) {
          continue;
      }
      p = q = NULL;
      if ((lstat(de->d_name, &st) != 0)) {
          ln_log(LNLOG_SERR, LNLOG_CARTICLE,
                "postarticles: cannot stat %s: %m",
                de->d_name);
          continue;
      }

      if (!S_ISREG(st.st_mode)) {
          ln_log(LNLOG_SNOTICE, LNLOG_CARTICLE,
                "postarticles: %s is not a regular file",
                de->d_name);
          movetofailed(de->d_name);
          continue;
      }

      if (!(st.st_mode & S_IRUSR)) {
          ln_log(LNLOG_SINFO, LNLOG_CARTICLE,
                "postarticles: skipping %s, not complete",
                de->d_name);
          continue;
      }

      f = fopen(de->d_name, "r");
      if (f == NULL) {
          ln_log(LNLOG_SERR, LNLOG_CARTICLE,
                "postarticles: cannot open file %s for reading: %m",
                de->d_name);
          movetofailed(de->d_name);
          continue;
      }

      p = fgetheader(f, "Newsgroups:");
      q = fgetheader(f, "Message-ID:");
      if (p == NULL || q == NULL) {
          ln_log(LNLOG_SERR, LNLOG_CARTICLE,
                "postarticles: file %s lacks Newsgroups "
                "and/or Message-ID header (this cannot happen)",
                de->d_name);
          movetofailed(de->d_name);
          goto free_cont;
      }

      if (!current_server->post_anygroup) {
          char *pp = critstrdup(p, "postarticles");
          if (!isgrouponserver(current_server, pp)) {
            ln_log(LNLOG_SINFO, LNLOG_CARTICLE,
                  "%s: postarticles: file %s: only_groups_pcre excluded "
                  "or server does not carry newsgroups %s",
                  current_server->name, de->d_name, p);
            free(pp);
            goto free_cont;
          }
          free(pp);
      }

      if ((haveid = ismsgidonserver(current_server, q)) != 1) {
          ln_log(LNLOG_SINFO, LNLOG_CARTICLE,
                "%s: postarticles: trying to post file %s Message-ID %s",
                current_server->name, de->d_name, q);

          xsnprintf(lineout, SIZE_lineout, "POST\r\n");
          putaline();
          r = nntpreply(current_server);
          if (r != 340) {
            char *e = lastreply();
            if (e == NULL) {
                ln_log(LNLOG_SERR, LNLOG_CARTICLE,
                      "%s: postarticles: server disconnect or timeout"
                      " while trying to post file %s)",
                      current_server->name, de->d_name);
                goto free_ret;
            }

            ln_log(LNLOG_SERR, LNLOG_CARTICLE,
                  "%s: postarticles: server replied \"%s\" to our POST command, skipping server",
                  current_server->name, e);
            goto free_ret;
          } else {
            /* server replied 340, it is willing to accept our article */
            int postok = 0;
            debug--;
            while ((line = getaline(f)) != NULL) {
                /* can't use putaline() here because
                   line length of lineout is restricted */
                if (line[0] == '.')
                  fputc('.', nntpout);
                fputs(line, nntpout);
                fputs("\r\n", nntpout);
            };
            fflush(nntpout);
            debug = debugmode;
            xsnprintf(lineout, SIZE_lineout, ".\r\n");
            putaline();
            line = mgetaline(nntpin);
            if (!line) {      /* timeout: posting failed */
                ln_log(LNLOG_SERR, LNLOG_CARTICLE,
                      "%s: postarticles: server disconnect or timeout"
                      " after sending article %s)",
                      current_server->name, de->d_name);
                goto free_ret;
            }
            line = critstrdup(line, "postarticles");
            if (strncmp(line, "240", 3) == 0) {
                postok = 1;
            } else if (ismsgidonserver(current_server, q)) {
                syslog(LOG_NOTICE,
                      "%s: postarticles: posting resulted in \"%s\", "
                      "but article is available upstream, assuming OK.",
                      current_server->name, line);
                printf("%s: postarticles: posting resulted in \"%s\",\n"
                      "but article is available upstream, assuming OK.",
                      current_server->name, line);
                postok = 1;
            }
            if (postok) {
                if (verbose > 2)
                  printf("%s: postarticles: POST of article %s OK\n",
                        current_server->name, de->d_name);
                n++;
                if (unlink(de->d_name)) {
                  ln_log(LNLOG_SERR, LNLOG_CARTICLE,
                         "postarticles: unable to unlink posted article %s: %m",
                         de->d_name);
                }
            } else {
                ln_log(LNLOG_SERR, LNLOG_CARTICLE,
                      "%s: postarticles: Article file %s Message-ID %s"
                      " was rejected: \"%s\"",
                      current_server->name, de->d_name, q, line);
                movetofailed(de->d_name);
            }
            free(line);
          }
          haveid = 0;
      } else if (haveid) {
          syslog(LOG_INFO, "%s: postarticles: Message-ID of %s already in use"
                " upstream -- article discarded\n", 
                current_server->name, de->d_name);
          if (verbose > 2)
            printf("%s: postarticles: %s already available upstream\n",
                  current_server->name, de->d_name);
          unlink(de->d_name);
      }
free_cont:
      fclose(f);
      if (p) free(p);
      if (q) free(q);
      continue;
free_ret:
      if (p) free(p);
      if (q) free(q);
      fchdir(savedir);
      close(savedir);
      closedir(d);
      return 0;
    } /* while de = readdir */
    closedir(d);
    if (verbose)
      printf("%s: %d article%s posted.\n", current_server->name, n, PLURAL(n));
    syslog(LOG_INFO, "%s: %d article%s posted.", current_server->name, n, PLURAL(n));
    fchdir(savedir);
    close(savedir);
    return 1;
}

static int
processupstream(const struct server *serv, time_t stamp)
{
    FILE *f;
    DIR *d;
    struct dirent *de;
    struct newsgroup *g;
    int havefile;
    unsigned long newserver = 0;
    char *l;
    char *oldfile;
    struct stat st1, st2;
    int have_st1 = 0;
    char s[SIZE_s+1];
    int aborting = 0;

    struct stringlist *ngs = NULL, *a, *b = NULL;

    /* read server info */
    formatserver(s, SIZE_s, serv, "");

    oldfile = critstrdup(s, "processupstream");
    havefile = 0;
    if ((f = fopen(s, "r")) != NULL) {
      if (fstat(fileno(f), &st1)) {
          int e = errno;
          ln_log(LNLOG_SERR, LNLOG_CSERVER, "Cannot stat %s: %s", s, strerror(e));
      } else {
          have_st1 = 1;
      }
      /* a sorted array or a tree would be better than a list */
      ngs = NULL;
      debug--;
      if (verbose > 1)
          printf("%s: reading server info from %s\n", serv->name, s);
      syslog(LOG_INFO, "%s: reading server info from %s", serv->name, s);
      while (((l = getaline(f)) != NULL) && (strlen(l))) {
          a = (struct stringlist *)critmalloc(sizeof(struct stringlist)
                                    + strlen(l),
                                    "Reading server info");
          strcpy(a->string, l);     /* RATS: ignore */
          a->next = NULL;
          if (ngs == NULL)
            ngs = a;
          else
            b->next = a;
          b = a;
      }
      havefile = 1;
      debug = debugmode;
      fclose(f);
    }

    xsnprintf(s, SIZE_s, "%s/interesting.groups", spooldir);
    d = opendir(s);
    if (!d) {
      ln_log(LNLOG_SERR, LNLOG_CSERVER, "opendir %s: %m", s);
      free(oldfile);
      return -1;
    }

    formatserver(s, SIZE_s, serv, "~");
    if (stat(s, &st2) == 0 && (!have_st1 || st2.st_mtime >
                         st1.st_mtime) && (f = fopen(s, "r"))) {
      int e = 0;

      /* roll in changes of a previous crash */
      syslog(LOG_INFO, "Merging in %s from previous run", s);
      if (verbose > 1)
          printf("Merging in %s from previous run\n", s);
      while (((l = getaline(f)) != NULL)) {
          char *p, *t = strchr(l, ' ');
          if (!t || !*++t)
            continue;
          (void)strtoul(t, &p, 10);
          if (p && !*p)
            replaceinlist(&ngs, l, (size_t)(t-l));
      }
      (void)fclose(f);  /* read-only file, assume no error */
      xsnprintf(s, SIZE_s, "%s/leaf.node/%s.new", spooldir, serv->name);
      f = fopen(s, "w");
      if (!f) {
          e = 1;
      } else {
          a = ngs;
          while (a && a->string && !ferror(f)) {
            (void)fputs(a->string, f);
            (void)fputc('\n', f);
            a = a->next;
          }
          if (fclose(f))
            e = 1;
      }
      if (e) {
          ln_log(LNLOG_SERR, LNLOG_CSERVER, "open %s: %m", s);
          (void)unlink(s);
          return -1;
      }
      if (rename(s, oldfile)) {
          ln_log(LNLOG_SERR, LNLOG_CSERVER, "rename %s -> %s: %m", s, oldfile);
          (void)unlink(s);
          return -1;
      }
    }

    formatserver(s, SIZE_s, serv, "~");
    (void)unlink(s);
    f = fopen(s, "w");
    if (f == NULL) {
      ln_log(LNLOG_SERR, LNLOG_CSERVER, "Could not open %s for writing: %s",
            s, strerror(errno));
    } else {
      /* make sure that at least SERVERINFO~ is complete */
      if (mysetvbuf(f, NULL, _IOLBF, 4096)) {
          /* try to at least use unbuffered then */
          mysetvbuf(f, NULL, _IONBF, 0);
      }
    }
    while ((de = readdir(d))) {
      if (isalnum((unsigned char)*(de->d_name))) {
          g = findgroup(de->d_name);
          if (g != NULL) {
            unsigned long newhigh;
            xsnprintf(s, SIZE_s, "%s ", g->name);
            newhigh = 1ul;
            l = havefile ? findinlist(ngs, s) : NULL;
            if (l && *l) {
                char *t;
                l = strchr(l, ' ');
                if (l) {
                  newhigh = strtoul(l, &t, 10);
                  if (t == l || *t)
                      newhigh = 1ul;
                }
            }
            newserver = getgroup(serv, g, newhigh);
            /* run this independent of delaybody mode, because
             * the admin may have switched delaybody off recently,
             * and we still want users to be able to retrieve
             * articles. */
            if (newserver) newhigh = newserver;
            if (f != NULL && newhigh > 0) {
                fprintf(f, "%s %lu\n", g->name, newhigh);
            }
            if (!newserver) {
                fprintf(stderr, "Warning: aborting fetch from %s due to previous condition.\n", serv->name);
                syslog(LOG_WARNING, "Warning: aborting fetch from %s due to previous condition.", serv->name);
                aborting = 1;
                break;
            }
          } else {
            if (verbose > 1)
                printf("%s not found in groupinfo file\n", de->d_name);
            syslog(LOG_NOTICE, "%s not found in groupinfo file", de->d_name);
          }
      } /* if isalnum */
    } /* while readdir */
    closedir(d);
    if (f != NULL) {
      int ren = 1;
      formatserver(s, SIZE_s, serv, "~");
      if (ferror(f))
          ren = 0;
      if (fflush(f))
          ren = 0;
      if (fclose(f))
          ren = 0;
      if (!aborting) {
          if (ren) {
            struct utimbuf ut;
            if (rename(s, oldfile)) {
                ln_log(LNLOG_SERR, LNLOG_CSERVER, "cannot rename %s to %s: %m",
                      s, oldfile);
            }
            ut.modtime = ut.actime = stamp;
            if (utime(oldfile, &ut)) {
                ln_log(LNLOG_SERR, LNLOG_CSERVER, "cannot set proper time for %s to %lu: %m",
                      s, (unsigned long)stamp);
            }
          } else {
            ln_log(LNLOG_SERR, LNLOG_CSERVER, "write error on %s, old version of %s kept",
                  s, oldfile);
          }
      }
    }

    free(oldfile);
    freelist(ngs);
    return 0;
}

/*
 * checks whether all newsgroups have to be retrieved anew
 * returns 0 if yes, time of last update if not
 * mtime is the time when active was fetched fully
 * atime is the time when active was last updated
 */
static time_t
checkactive(void)
{
    struct stat st;
    char *s = activeread();

    if (stat(s, &st)) {
      free(s);
      return 0;
    }
    if ((now - st.st_mtime) < (timeout_active * SECONDS_PER_DAY)) {
      if (debugmode)
          syslog(LOG_DEBUG,
               "Last LIST done %d seconds ago: NEWGROUPS\n",
               (int)(now - st.st_mtime));
      free(s);
      return st.st_atime;
    } else {
      if (debugmode)
          syslog(LOG_DEBUG, "Last LIST done %d seconds ago: LIST\n",
               (int)(now - st.st_mtime));
      free(s);
      return 0;
    }
}

static int
updateactive(void)
{
    struct stat st;
    struct utimbuf buf;
    FILE *f;
    char *s = activeread();
    int rc = 0;

    if (stat(s, &st)) {
      /* active.read probably doesn't exist */
      (void)unlink(s); /* delete it in case it's junk */
      if ((f = fopen(s, "w")) != NULL) {
          if (fsync(fileno(f))) rc = -1;
          if (fclose(f)) rc = -1;
      } else {
          /* f == NULL, open error */
          rc = -1;
      }
    } else {
      buf.actime = (now < st.st_atime) ? st.st_atime : now;
      /* now < update may happen through HW failures */
      buf.modtime = st.st_mtime;
      if (utime(s, &buf)) {
          rc = -1;
      }
    }
    if (rc)
      ln_log(LNLOG_SERR, LNLOG_CTOP, "cannot update or create %s: %m", s);
    free(s);
    return rc;
}

static void
error_refetch(const char *e) {
    ln_log(LNLOG_SERR, LNLOG_CTOP,
          "ERROR: FETCHNEWS MUST REFETCH THE WHOLE ACTIVE FILE NEXT RUN.");
    ln_log(LNLOG_SERR, LNLOG_CTOP, "REASON: %s", e);
}

int
main(int argc, char **argv)
{
    /* the volatile keyboard avoids clobbering by siglongjmp */
    volatile time_t lastrun;
    volatile int rc = 0, skip_servers = 0;
    volatile int anypost = 0, waitchild = 0, quiet;
    struct server *current_server;
    volatile int need_refetch = 0;

    int option, reply;
    pid_t pid;

    verbose = quiet = 0;
    postonly = waitchild = 0;

    myopenlog("fetchnews");

    if (!initvars(argv[0]))
      exit(1);

    while ((option = getopt(argc, argv, "Pfhlnvx:qw")) != -1) {
      if (option == 'v') {
          verbose++;
          quiet = 0;
      } else if (option == 'h') {
          usage();
          exit(EXIT_SUCCESS);
      } else if (option == 'x') {
          char *nptr, *endptr;
          nptr = optarg;
          endptr = NULL;
          extraarticles = strtoul(nptr, &endptr, 0);
          if (!nptr || !*nptr || !endptr || *endptr || !extraarticles) {
            usage();
            exit(1);
          }
          syslog(LOG_NOTICE, "fetchnews: run with option -x %lu", extraarticles);
      } else if (option == 'l') {
          usesupplement = 0;  /* don't use supplementary servers */
      } else if (option == 'n') {
          noexpire = 1;
      } else if (option == 'f') {
          forceactive = 1;
      } else if (option == 'P') {
          postonly = 1;
      } else if (option == 'q') {
          verbose = 0;
          quiet = 1;
      } else if (option == 'w') {
          waitchild = 1;
      } else {
          usage();
          exit(1);
      }
    }

    /* Set line buffering to ensure that logging gets displayed promptly. */
    if (mysetvbuf(stdout, NULL, _IOLBF, 4096)) {
      /* Try to at least use unbuffered then */
      mysetvbuf(stdout, NULL, _IONBF, 0);
    }

    now = time(NULL);

    umask(2);

    if (!readconfig(0)) {
      fprintf(stderr, "Reading configuration failed, exiting "
             "(see syslog for more information).\n");
      freeconfig();
      exit(1);
    }

    syslog(LOG_DEBUG, "leafnode %s: verbosity level is %d, debugmode is %d",
          version, verbose, debugmode);
    if (verbose || debugmode) {
      printf("leafnode %s: verbosity level is %d, debugmode is %d\n",
            version, verbose, debugmode);
      if (verbose > 1 && noexpire) {
          printf("Don't automatically unsubscribe unread newsgroups.\n");
      }
    }

    if (forceactive) {
      ln_log(LNLOG_SNOTICE, LNLOG_CTOP,
            "Forced active fetch requested from command-line (option -f).");
    }

    if (try_lock(timeout_lock)) {
      ln_log(LNLOG_SERR, LNLOG_CTOP,
            "Cannot obtain lock file, aborting.");
      freeconfig();
      exit(1);
    }

    if (!postonly) {
      readactive();
      if (!active) {
          fakeactive(); /* we need proper lowwater/highwater marks
                       for the groups that exist */
          ln_log(LNLOG_SNOTICE, LNLOG_CTOP,
                "Forced active fetch after trouble reading active file.");
          forceactive = 1;
      }

      readfilter(filterfile);
    }

    lastrun = 0;
    if (!postonly && !forceactive) {
      lastrun = checkactive();
      if (!lastrun) {
          ln_log(LNLOG_SNOTICE, LNLOG_CTOP, "Active has not been fetched completely in previous run");
          ln_log(LNLOG_SNOTICE, LNLOG_CTOP, "or has never been fetched, forcing active fetch.");
          forceactive = 1;
      }
    }

    if (!postonly && !forceactive) {
      int staterror;
      struct stat st;
      char s[SIZE_s+1];

      xsnprintf(s, SIZE_s, "%s/leaf.node/groupinfo", spooldir);
      if ((staterror = stat(s, &st)) && errno != ENOENT) {
          /* this should happen only when the problem occurs after
           * readactive() above, but needs to be checked nonetheless */
          ln_log(LNLOG_SERR, LNLOG_CTOP, "Cannot open %s: %m", s);
          unlink(lockfile);
          freeconfig();
          exit(EXIT_FAILURE);
      }
      if (staterror || st.st_size < 7) {
          ln_log(LNLOG_SNOTICE, LNLOG_CTOP,
                "Groupinfo file %s not present or too small, "
                "forcing active fetch.", s);
          forceactive = 1;
      }
    }

    if (forceactive) {
      oldactive = active;
      oldactivesize = activesize;
      active = NULL;
      activesize = 0;
      if (killactiveread()) exit(1);
    }

    if (mysigact(SIGINT, 0, sig_int, SIGALRM) != 0)
      fprintf(stderr, "Can't catch SIGINT.\n");
    if (mysigact(SIGTERM, 0, sig_int, SIGALRM) != 0) 
      fprintf(stderr, "Can't catch SIGTERM.\n");
    if (mysigact(SIGPIPE, 0, sig_int, SIGALRM) != 0)
      fprintf(stderr, "Can't catch SIGPIPE.\n");
    {
      int sig = sigsetjmp(jmpbuffer, 1);
      
      if (sig != 0) {
          ln_log(LNLOG_SWARNING, LNLOG_CTOP,
                "fetchnews: caught signal %d, shutting down.", sig);
          nntpquit();
          if (!rc)
            rc = 2;
          if (forceactive) {
            error_refetch("caught signal that caused a premature abort.");
            need_refetch = 1;
          }
          skip_servers = 1;   /* in this case, jump the while ... loop */
      } else {
          canjump = 1;
      }
    }

    /* remove groups that haven't been read in a long time */
    if (!noexpire)
      expire_interesting();

    mgetaline_settimeout(timeout_fetchnews);

    /* main server loop */
    for (current_server = servers; 
                !skip_servers && current_server; 
                current_server = current_server->next) {
      if (verbose) {
          if (current_server->port)
            printf("%s: connecting to port %u...\n",
                   current_server->name, current_server->port);
          else
            printf("%s: connecting to port nntp...\n",
                   current_server->name);
      }
      fflush(stdout);
      reply = nntpconnect(current_server);
      if (reply) {
          int r2;
          if (verbose) {
            int namlen = strlen(current_server->name);
            printf("%s: connected.\n", current_server->name);
            if (stat_is_evil)
                printf("%s: server software does not implement\n"
                     "%*s  STAT <message-ID> properly,\n"
                     "%*s  using workaround with HEAD instead,\n"
                     "%*s  at the expense of bandwidth.\n",
                     current_server->name, namlen, "", namlen, "", namlen, "");
            else
                printf("%s: using STAT <message-ID> command.\n",
                      current_server->name);
          }

          if (current_server->username)
            if (!authenticate(current_server) && current_server->password)
                ln_log(LNLOG_SERR, LNLOG_CTOP,
                      "error: may have been caused by premature authentication and be rather harmless.");

          /* Get INN's nnrpd on the phone */
          xsnprintf(lineout, SIZE_lineout, "MODE READER\r\n");
          putaline();
          r2 = nntpreply(current_server);
          if (r2 < 400) reply = r2;

          if (reply == 498) {
            ln_log(LNLOG_SERR, LNLOG_CTOP, "%s: protocol error after sending mode reader",
                   current_server->name);
          } else {
            if (reply == 200 && current_server->nopost == 0) {
                anypost = 1;
                (void)postarticles(current_server);
            } else if (verbose) {
                printf("Not posting to %s: ", current_server->name);
                if (reply != 200)
                  printf("non-permission ");
                if (current_server->nopost)
                  printf("nopost-set ");
                printf("\n");
            }
            if (!date_is_evil)
                check_date(current_server);
            if (!postonly && !current_server->noread) {
                time_t stamp = lastrun;

                /* get list of newsgroups or new newsgroups */
                if (current_server->updateactive) {
                  int r;
                  if ((r = nntpactive(current_server, &stamp))) {
                      if (forceactive || r == -2) {
                        error_refetch("obtaining the active file failed.");
                        need_refetch = 1;
                      }
                      rc = 1;
                  }
                } else {
                  if (verbose)
                      printf("%s: not attempting to update newsgroups list\n",
                           current_server->name);
                }

                processupstream(current_server, stamp);
            }
          }
          nntpquit();
          if (verbose)
            printf("%s: conversation completed, disconnected.\n", current_server->name);
      } else { /* reply = nntpconnect */
          if (verbose)
            printf("%s: connection failed.\n", current_server->name);
          if (forceactive && current_server->updateactive) {
            error_refetch("fetching the active list from a server failed.");
            need_refetch = 1;
          }
          rc = 2;
      }
      if (!usesupplement)
          break;
    }
    mergegroups(); /* just in case we were interrupted while downloading
                  the list. */

    (void)mysigact(SIGINT, 0, SIG_IGN, 0);      /* do not siglongjmp any more */
    (void)mysigact(SIGTERM, 0, SIG_IGN, 0);     /* do not siglongjmp any more */
    (void)mysigact(SIGPIPE, 0, SIG_IGN, 0);     /* SIGPIPE should not happen below */

    if (rc != 0 && oldactive) {
      /* restore old active data to keep low/high marks */
      unsigned long i;
      for (i = 0; i < oldactivesize; i++) {
          insertgroup(oldactive[i].name, 0, 0, 0);
      }
      mergegroups();
      killactiveread();
    }

    freeactive(oldactive);
    oldactive = NULL;
    oldactivesize = 0;

    if (anypost == 0 && skip_servers == 0 && rc != 2) {
      const char *e = "WARNING: found no server with posting permission!";
      if (!quiet)
          fprintf(stderr, "%s\n", e);
      syslog(LOG_WARNING, "%s", e);
    }

    if (rc == 2) {
      const char *e = "WARNING: some servers have not been queried!";
      if (!quiet)
          fprintf(stderr, "%s\n", e);
      syslog(LOG_WARNING, "%s", e);
    }

    if (fflush(stdout)) {
      /* to avoid double logging of stuff */
      ln_log(LNLOG_SERR, LNLOG_CTOP, "cannot flush standard output: %m");
    }

    {
      /* OK, we do a pipe trick to hold the child until the parent
       * handed over the lock file (this must happen so it does not
       * inadvertently get removed as stale). The child reads from the
       * pipe and is blocked until the parent has written to the pipe.
       * The parent writes to the pipe after it has handed over the
       * lock.
       */
      int pfd[2], pipeok;

      pipeok = pipe(pfd);

      if (!postonly) {
          /* only update active.read when have read active from all servers */
          if (writeactive()) {
            ln_log(LNLOG_SERR, LNLOG_CTOP, "Error writing groupinfo.");
            error_refetch("cannot write groupinfo file.");
            rc = 1;
            /* mark for refetch */
            killactiveread();
          } else {
            /* active written successfully */
            if (updateactive()) {
                error_refetch("cannot update active.read file.");
                need_refetch = 1;
                rc = 1;
            }

            if (need_refetch) {
                /* mark for refetch */
                killactiveread();
            }
          }

#ifdef HAVE_WORKING_FORK
          pid = waitchild ? -1 : fork();
#else
          pid = -1; waitchild = 1;
#endif

          switch (pid) {
            case -1:
                if (!waitchild)
                  syslog(LOG_NOTICE, "fork: %m, running on parent schedule.");
                if (verbose)
                  printf("updating overview data in the foreground...\n");
                fixxover();
                unlink(lockfile);
                if (verbose)
                  printf("done.\n");
                break;

            case 0:
                (void)setsid();
                if (debugmode)
                  syslog(LOG_DEBUG, "Process forked.");
                if (pipeok == 0) {
                  /* wait for parent to hand over the lock */
                  char buf[4];
                  (void)close(pfd[1]);
                  /* we don't REALLY check for errors here, the worst thing
                   * that could happen (in case of a kernal bug...) to
                   * us is that we start early and delete the parent's
                   * lock file before the parent handed it over -- but
                   * at that time, we'll then exit and everything is
                   * in order, with just one bogus message in the log.
                   */
                  while (read(pfd[0], buf, sizeof(buf)) < 0) {
                      if (errno != EAGAIN && errno != EINTR)
                        break;
                  }
                  (void)close(pfd[0]); }

                  fixxover();
                  freeactive(active);
                  if (unlink(lockfile))
                      ln_log(LNLOG_SERR, LNLOG_CTOP,
                            "unlink(\"%s\"): %m", lockfile);
                  if (debugmode)
                      syslog(LOG_DEBUG, "Process done.");
                  freeconfig();
                  _exit(0);
                  break;

            default:
                  {
                      int lock_ok = handover_lock(pid);

                      if (verbose) puts("Started process to update overview data in the background.\nNetwork activity has finished.");
                      if (pipeok == 0) {
                        /* tell child it has the lock */
                        (void)close(pfd[0]);
                        (void)writes(pfd[1], "GO");
                        (void)close(pfd[1]);
                      }

                      if (lock_ok) {
                        /* could not hand over lock file to child, so wait until it
                           dies */
                        syslog(LOG_NOTICE, "could not hand over lockfile to child %lu: %m, "
                              "waiting until child is done.",
                              (unsigned long)pid);
                        (void)waitpid(pid, NULL, 0);
                      } else {
                        syslog(LOG_INFO, "child has process ID %lu",
                              (unsigned long)pid);
                      }
                      break;
                  }
          }
      } else { /* if (postonly) */
          unlink(lockfile);
      }
    }

    freeactive(active);
    freeconfig();
    exit(rc);
}

Generated by  Doxygen 1.6.0   Back to index