#!/usr/bin/perl -w

eval 'exec /usr/bin/perl -w -S $0 ${1+"$@"}'
  if 0; # not running under some shell

our $Version="0.2";

use strict;
use Getopt::Long;
use File::Temp qw/ tempfile /;

# Display mailman2lurker's version number.
sub version {
  print "mailman2lurker version $Version\n";
  exit;
}

# Display usage help.
sub usage() {
    print STDERR <<EOF;
mailman2lurker - Import mailman lists and their archives into a lurker database

Usage: mailman2lurker -p|-i [options]

Options:
  -c, --config <config>    use this conffile (default: /etc/lurker/lurker.conf)
  -i, --import             import list archives from mailman's mbox files
     -f, --listfile <file>     import only lists in <file>.
     -d, --lists <one,two,..>  import only lists that are given as argument.
  -p, --parse              create a lurker conf include from mailman lists
     -o, --outfile <outfile>   write to this file. (default: <config>.mailman)
     -g, --group <group>       use this group for lists. (default: undef)
     -a, --listhost <host>     use this host for lists. (default: example.org)
     -l, --link <url link>     use for the url link. (default: listurl)
     -s, --lang <language>     use this language for new lists. (default: en)
     -w, --write-lists <file>  write a list of parsed lists to <file>.
     -x, --list-cmd <cmd|file> use this command/file as list of mailman lists.
                               (default: "/usr/lib/mailman/bin/list_lists |")
  -h, --help               Display this help message.
  -v, --version            Display mailman2lurker's version number.

EOF
    exit 1;
}

sub check_group {
  my ($group) = @_;
  my $lurkergroups = "/usr/bin/lurker-list -g |";
  open LURKERGROUPS, $lurkergroups or die "Could not open $lurkergroups: $!";
  while (<LURKERGROUPS>) {
    if (/^$group$/) {
      print STDERR "Group $group already exists in lurker configuration, you need to configure another one with -g.\n";
      exit 1;
    }
  }
  close LURKERGROUPS;
}

sub lurker_lists {
  my ($list, @lists);
  my $lurkerlist = "/usr/bin/lurker-list -c $glob::lurkerconf -i |";
  open LURKERLIST, $lurkerlist or die "Could not open $lurkerlist: $!";
  while (<LURKERLIST>) {
    chomp;
    if (($list) = /^([0-9-_\w]*)$/) {
      push @lists, $list;
    }
  }
  close LURKERLIST;
  return @lists;
}

sub mailman_lists {
  my ($list_cmd, $defgroup, $listhost, $listurl, $lang) = @_;
  my ($list, $desc, %mmlists);
  my ($fh, $filename) = tempfile();
  open my $ll_exec, $list_cmd or die "Could not open $list_cmd: $!";
  while (<$ll_exec>) { print $fh $_; };
  close $ll_exec;
  close $fh;

  my $ll_charset = uc(`/usr/bin/file -i $filename | sed -e 's/^.*charset=//g'`);
  chomp ($ll_charset);

  my @list_lists = `/usr/bin/iconv -f $ll_charset -t UTF-8 $filename`;
  unlink($filename);

  for (@list_lists) {
    chomp;
    next unless (($list, $desc) = /^\s*([0-9-_\w]*)\s-\s(.*)\s*$/);
    if ($desc eq "[no description available]") {
      $desc = "";
    }
    my $list_lc = lc($list);
    if (not (grep(/$list_lc/, @glob::lurkerlists))) {
      $mmlists{$list_lc} = { id => $list_lc,
			    group => $defgroup,
			    title => $list,
			    address => $list . '@' . $listhost,
			    link => $listurl . '/' . $list,
			    desc => $desc,
			    lang => $lang };
    }
  }
  my $mmlref = \%mmlists;
  return $mmlref;
}

sub dump_lists {
  my ($dlref) = shift;
  my %dlists = %$dlref;
  my ($group, $writeflag) = "";
  open LOCALCONF, ">> $glob::localconf" or die "Could not open $glob::localconf: $!";
  if ($glob::writefile ne "") {
    $writeflag = "yes";
    open WRITEFILE, "> $glob::writefile" or die "Could not open $glob::writefile: $!";
  }
  #for $_ (sort { $dlists{$a}{id} cmp $dlists{$b}{id} } keys %dlists) {
  for $_ (sort {($dlists{$a}{group} cmp $dlists{$b}{group} ) or ($dlists{$a}{id} cmp $dlists{$b}{id} ) } keys %dlists) {
    unless ($dlists{$_}{group} eq $group)
    {
      $group = $dlists{$_}{group};
      print LOCALCONF "\n";
      print LOCALCONF "group = $group\n";
      print LOCALCONF "        heading = $group\n";
      print LOCALCONF "\n";
    }
    print LOCALCONF "        list = $dlists{$_}{id}\n";
    print LOCALCONF "                title = $dlists{$_}{title}\n";
    print LOCALCONF "                language = $dlists{$_}{lang}\n";
    print LOCALCONF "                address = $dlists{$_}{address}\n";
    print LOCALCONF "                link = $dlists{$_}{link}\n";
    print LOCALCONF "                description = $dlists{$_}{desc}\n";
    print LOCALCONF "\n";
    if ($writeflag) { print WRITEFILE "$dlists{$_}{id}\n"; }
  }
  if ($writeflag) { close WRITEFILE; }
  close LOCALCONF;
}

sub import_lists {
  my $iulref = shift;
  my @ulists = @$iulref;
  my $ulfile = pop(@ulists);
  my @lists;
  if ($ulfile ne " ") {
    if (-f $ulfile) {
      open ULFILE, $ulfile or die "Could not open $ulfile: $!";
      while (<ULFILE>) {
        chomp;
	push @lists, $_;
      }
      close ULFILE;
    }
    else {
      print STDERR "file $ulfile does not exist.\n";
      exit 1;
    }
  }
  elsif (@ulists) {
    @lists = @ulists;
  }
  else {
    @lists = @glob::lurkerlists;
  }
  for my $list (@lists) {
    my $mbox = "/var/lib/mailman/archives/private/$list.mbox/$list.mbox";
    if (-f $mbox) {
      if(system("/usr/bin/lurker-index", "-c", "$glob::lurkerconf",  "-l", "$list", "-i", "$mbox") != 0) {
        print STDERR "archives for $list could not be imported.\n";
      }
    }
    else {
      print STDERR "no archives available for $list at $mbox.\n";
    }
  }
}

my ($listhost, $defgroup, $listurl, $language, $ll_cmd) =
		("example.org", "undef", "http://example.org", "en",
		 "/usr/lib/mailman/bin/list_lists |");
my ($import, $parse, $uselistfile, @uselists);
$glob::lurkerconf = "/etc/lurker/lurker.conf";
$glob::localconf = "$glob::lurkerconf.mailman";
$glob::writefile = "";
@glob::lurkerlists = ();
my @list_props = qw/title address link language description/;

# Bundling is nice anyway, and it is required or Getopt::Long will confuse
# -V and -v.
Getopt::Long::Configure("bundling");

GetOptions(
  'a|listhost=s'    => \$listhost,
  'i|import'        => sub { $import=1 },
  'f|listfile=s'    => \$uselistfile,
  'd|lists=s'       => \@uselists,
  'c|config=s'      => \$glob::lurkerconf,
  'g|group=s'       => \$defgroup,
  'h|help'          => \&usage,
  'l|link=s'        => \$listurl,
  'o|outfile=s'     => \$glob::localconf,
  'p|parse'         => sub { $parse=1 },
  's|language=s'    => \$language,
  'v|version'       => \&version,
  'w|write-lists=s' => \$glob::writefile,
  'x|list-cmd=s'    => \$ll_cmd );
@uselists = split(/,/,join(',',@uselists));

@glob::lurkerlists = lurker_lists();

if ($parse && $import) {
  print STDERR "--parse and --import cannot be used together.\n";
  exit 1;
}
elsif ($parse) {
  check_group($defgroup);
  my $mmmlref = mailman_lists($ll_cmd, $defgroup, $listhost, $listurl, $language);
  dump_lists($mmmlref);
}
elsif ($import && $uselistfile && @uselists) {
  print STDERR "--listfile and --lists cannot be used together.\n";
  exit 1;
}
elsif ($import) {
  if (not ($uselistfile)) { $uselistfile = " "; }
  push @uselists, $uselistfile;
  my $ulref = \@uselists;
  import_lists($ulref);
}
else {
  usage();
}

__END__

# to create a manpage for mailman2lurker, invoke the following command:
# pod2man --section=8 --center="Administrative commands" --release="" mailman2lurker mailman2lurker.8

=head1 NAME

mailman2lurker - Import mailman lists and their archives into a lurker database.

=head1 SYNOPSIS

Z<> B<mailman2lurker> B<-i>|B<-p> [I<options>]

=head1 DESCRIPTION

B<mailman2lurker> has mainly two functions. First, it parses mailman's lists
and creates a lurker configuration include out of them. Second, it imports
archives from mailman lists into a lurker database.

To build a new lurker database from mailmans archives, first run
'B<mailman2lurker -p>'. Next you need to add 'B<include =
/etc/lurker/lurker.conf.mailman>' into /etc/lurker/lurker.conf.local.
Finally import mailmans mboxes by running 'B<mailman2lurker -i>'.

If you want to add new lists to an already existing lurker configuration, run
'B<mailman2lurker -p -w newlists>' and afterwards 'B<mailman2lurker -i -f
newlists>'. This will import only the mbox files of mailman lists that have been
added to the lurker configuration because they were new to lurker.

=head1 COMMANDS

=over 4

=item B<-p>, B<--parse>

Parse mailman lists, and create a lurker configuration include file containing
all parsed lists. Lists that are already configured in lurker are ignored.

=item B<-i>, B<--import>

Import archives of all lists. This parses the lurker configuration and imports
the mailman mbox at /var/lib/mailman/archives/private/<list>.mbox/<list>.mbox
for every list.

=back

=head1 OPTIONS

=over 4

=item B<-c>, B<--config> <I<config>>

Use <I<config>> as lurker configuration file. Default: /etc/lurker/lurker.conf

=item B<-o>, B<--outfile> <I<outfile>>

Write list configuration to <I<outfile>>. If this is not specified,
<lurker.conf>.mailman will be used.
This option is only useful in conjunction with B<-p>.

=item B<-g>, B<--group> <I<group>>

Use <I<group>> for new lists. Default is 'undef'.
This option is only useful in conjunction with B<-p>.

=item B<-a>, B<--listhost> <I<listhost>>

Use <I<listhost>> as host for new list addresses. If a new list is called
'mylist' and <I<listhost>> is lists.org, then the list address will be
mylist@lists.org. Default is 'example.org'.
This option is only useful in conjunction with B<-p>.

=item B<-l>, B<--link> <I<url link>>

Use <I<url link>> as base for links of new lists. If a list is called 'mylist',
and <I<url link>> is 'http://lists.org', then the link address will be
http://lists.org/mylist. Default is 'http://example.org'.
This option is only useful in conjunction with B<-p>.

=item B<-s>, B<--lang> <I<language>>

Use <I<language>> as language for new lists. Needs to be a language token.
Default is 'en'. This option is only useful in conjunction with B<-p>.

=item B<-w>, B<--write-lists> <I<file>>

Write a list of all lists that have been added to the lurker configuration.
Safe that list in <I<file>>. Default is off.
This option is only useful in conjunction with B<-p>.

=item B<-x>, B<--list-cmd> <I<file>>

Use <I<command|file>> to get the list of mailman lists that should be parsed.
When <I<command|file>> is a command, it needs to end with ' |'.
Default is '/var/lib/mailman/bin/list_lists |'.
This option is only useful in conjunction with B<-p>.

=item B<-f>, B<--listfile> <I<file>>

Import only lists that are listed in <I<file>>.
This option is only useful in conjunction with B<-i>.

=item B<-d>, B<--lists> <I<one>[,I<two>,I<...>]>

Import only lists that are given as argument.
This option is only useful in conjunction with B<-i>.

=item B<-h>, B<--help>

Display a help message.

=item B<-v>, B<--version>

Display mailman2lurker's version number.

=back

=head1 SEE ALSO

L<list_lists(8)>, L<lurker-index(8)>

=head1 AUTHOR

This program was written by Jonas Meurer <jonas@freesources.org>.

=head1 COPYRIGHT

mailman2lurker may be copied and modified under the terms of the GNU General
Public License.
