<html><head><meta name="color-scheme" content="light dark"></head><body><pre style="word-wrap: break-word; white-space: pre-wrap;">package CPANPLUS::Internals::Source;

use strict;

use CPANPLUS::Error;
use CPANPLUS::Module;
use CPANPLUS::Module::Fake;
use CPANPLUS::Module::Author;
use CPANPLUS::Internals::Constants;

use File::Fetch;
use Archive::Extract;

use IPC::Cmd                    qw[can_run];
use File::Temp                  qw[tempdir];
use File::Basename              qw[dirname];
use Params::Check               qw[check];
use Module::Load::Conditional   qw[can_load];
use Locale::Maketext::Simple    Class =&gt; 'CPANPLUS', Style =&gt; 'gettext';

$Params::Check::VERBOSE = 1;

### list of methods the parent class must implement
{   for my $sub ( qw[_init_trees _finalize_trees
                     _standard_trees_completed _custom_trees_completed
                     _add_module_object _add_author_object _save_state
                    ]
    ) {
        no strict 'refs';
        *$sub = sub {
            my $self    = shift;
            my $class   = ref $self || $self;

            require Carp;
            Carp::croak( loc( "Class %1 must implement method '%2'",
                              $class, $sub ) );
        }
    }
}

{
    my $recurse; # flag to prevent recursive calls to *_tree functions

    ### lazy loading of module tree
    sub _module_tree {
        my $self = $_[0];

        unless ($self-&gt;_mtree or $recurse++ &gt; 0) {
            my $uptodate = $self-&gt;_check_trees( @_[1..$#_] );
            $self-&gt;_build_trees(uptodate =&gt; $uptodate);
        }

        $recurse--;
        return $self-&gt;_mtree;
    }

    ### lazy loading of author tree
    sub _author_tree {
        my $self = $_[0];

        unless ($self-&gt;_atree or $recurse++ &gt; 0) {
            my $uptodate = $self-&gt;_check_trees( @_[1..$#_] );
            $self-&gt;_build_trees(uptodate =&gt; $uptodate);
        }

        $recurse--;
        return $self-&gt;_atree;
    }

}


=pod

=head1 NAME

CPANPLUS::Internals::Source - internals for updating source files

=head1 SYNOPSIS

    ### lazy load author/module trees ###

    $cb-&gt;_author_tree;
    $cb-&gt;_module_tree;

=head1 DESCRIPTION

CPANPLUS::Internals::Source controls the updating of source files and
the parsing of them into usable module/author trees to be used by
C&lt;CPANPLUS&gt;.

Functions exist to check if source files are still C&lt;good to use&gt; as
well as update them, and then parse them.

The flow looks like this:

    $cb-&gt;_author_tree || $cb-&gt;_module_tree
        $cb-&gt;_check_trees
            $cb-&gt;__check_uptodate
                $cb-&gt;_update_source
            $cb-&gt;__update_custom_module_sources
                $cb-&gt;__update_custom_module_source
        $cb-&gt;_build_trees
            ### engine methods
            {   $cb-&gt;_init_trees;
                $cb-&gt;_standard_trees_completed
                $cb-&gt;_custom_trees_completed
            }
            $cb-&gt;__create_author_tree
                ### engine methods
                { $cb-&gt;_add_author_object }
            $cb-&gt;__create_module_tree
                $cb-&gt;__create_dslip_tree
                ### engine methods
                { $cb-&gt;_add_module_object }
            $cb-&gt;__create_custom_module_entries

    $cb-&gt;_dslip_defs

=head1 METHODS

=cut

=pod

=head2 $cb-&gt;_build_trees( uptodate =&gt; BOOL, [use_stored =&gt; BOOL, path =&gt; $path, verbose =&gt; BOOL] )

This method rebuilds the author- and module-trees from source.

It takes the following arguments:

=over 4

=item uptodate

Indicates whether any on disk caches are still ok to use.

=item path

The absolute path to the directory holding the source files.

=item verbose

A boolean flag indicating whether or not to be verbose.

=item use_stored

A boolean flag indicating whether or not it is ok to use previously
stored trees. Defaults to true.

=back

Returns a boolean indicating success.

=cut

### (re)build the trees ###
sub _build_trees {
    my ($self, %hash)   = @_;
    my $conf            = $self-&gt;configure_object;

    my($path,$uptodate,$use_stored,$verbose);
    my $tmpl = {
        path        =&gt; { default =&gt; $conf-&gt;get_conf('base'), store =&gt; \$path },
        verbose     =&gt; { default =&gt; $conf-&gt;get_conf('verbose'), store =&gt; \$verbose },
        uptodate    =&gt; { required =&gt; 1, store =&gt; \$uptodate },
        use_stored  =&gt; { default =&gt; 1, store =&gt; \$use_stored },
    };

    my $args = check( $tmpl, \%hash ) or return;

    $self-&gt;_init_trees(
        path        =&gt; $path,
        uptodate    =&gt; $uptodate,
        verbose     =&gt; $verbose,
        use_stored  =&gt; $use_stored,
    ) or do {
        error( loc("Could not initialize trees" ) );
        return;
    };

    ### return if we weren't able to build the trees ###
    return unless $self-&gt;_mtree &amp;&amp; $self-&gt;_atree;

    ### did we get everything from a stored state? if not,
    ### process them now.
    if( not $self-&gt;_standard_trees_completed ) {

        ### first, prep the author tree
        $self-&gt;__create_author_tree(
                uptodate    =&gt; $uptodate,
                path        =&gt; $path,
                verbose     =&gt; $verbose,
        ) or return;

        ### and now the module tree
        $self-&gt;_create_mod_tree(
                uptodate    =&gt; $uptodate,
                path        =&gt; $path,
                verbose     =&gt; $verbose,
        ) or return;
    }

    ### XXX unpleasant hack. since custom sources uses -&gt;parse_module, we
    ### already have a special module object with extra meta data. that
    ### doesn't gelwell with the sqlite storage engine. So, we check 'normal'
    ### trees from separate trees, so the engine can treat them differently.
    ### Effectively this means that with the SQLite engine, for now, custom
    ### sources are continuously reparsed =/ -kane
    if( not $self-&gt;_custom_trees_completed ) {

        ### update them if the other sources are also deemed out of date
        if( $conf-&gt;get_conf('enable_custom_sources') ) {
            $self-&gt;__update_custom_module_sources( verbose =&gt; $verbose )
                or error(loc("Could not update custom module sources"));
        }

        ### add custom sources here if enabled
        if( $conf-&gt;get_conf('enable_custom_sources') ) {
            $self-&gt;__create_custom_module_entries( verbose =&gt; $verbose )
                or error(loc("Could not create custom module entries"));
        }
    }

    ### give the source engine a chance to wrap up creation
    $self-&gt;_finalize_trees(
        path        =&gt; $path,
        uptodate    =&gt; $uptodate,
        verbose     =&gt; $verbose,
        use_stored  =&gt; $use_stored,
    ) or do {
        error(loc( "Could not finalize trees" ));
        return;
    };

    ### still necessary? can only run one instance now ###
    ### will probably stay that way --kane
#     my $id = $self-&gt;_store_id( $self );
#
#     unless ( $id == $self-&gt;_id ) {
#         error( loc("IDs do not match: %1 != %2. Storage failed!", $id, $self-&gt;_id) );
#     }

    return 1;
}

=pod

=head2 $cb-&gt;_check_trees( [update_source =&gt; BOOL, path =&gt; PATH, verbose =&gt; BOOL] )

Retrieve source files and return a boolean indicating whether or not
the source files are up to date.

Takes several arguments:

=over 4

=item update_source

A flag to force re-fetching of the source files, even
if they are still up to date.

=item path

The absolute path to the directory holding the source files.

=item verbose

A boolean flag indicating whether or not to be verbose.

=back

Will get information from the config file by default.

=cut

### retrieve source files, and returns a boolean indicating if it's up to date
sub _check_trees {
    my ($self, %hash) = @_;
    my $conf          = $self-&gt;configure_object;

    my $update_source;
    my $verbose;
    my $path;

    my $tmpl = {
        path            =&gt; { default =&gt; $conf-&gt;get_conf('base'),
                             store =&gt; \$path
                        },
        verbose         =&gt; { default =&gt; $conf-&gt;get_conf('verbose'),
                             store =&gt; \$verbose
                        },
        update_source   =&gt; { default =&gt; 0, store =&gt; \$update_source },
    };

    my $args = check( $tmpl, \%hash ) or return;

    ### if the user never wants to update their source without explicitly
    ### telling us, shortcircuit here
    return 1 if $conf-&gt;get_conf('no_update') &amp;&amp; !$update_source;

    ### a check to see if our source files are still up to date ###
    msg( loc("Checking if source files are up to date"), $verbose );

    my $uptodate = 1; # default return value

    for my $name (qw[auth dslip mod]) {
        for my $file ( $conf-&gt;_get_source( $name ) ) {
            $self-&gt;__check_uptodate(
                file            =&gt; File::Spec-&gt;catfile( $path, $file ),
                name            =&gt; $name,
                update_source   =&gt; $update_source,
                verbose         =&gt; $verbose,
            ) or $uptodate = 0;
        }
    }

    ### if we're explicitly asked to update the sources, or if the
    ### standard source files are out of date, update the custom sources
    ### as well
    ### RT #47820: Don't try to update custom sources if they are disabled
    ### in the configuration.
    $self-&gt;__update_custom_module_sources( verbose =&gt; $verbose )
        if $conf-&gt;get_conf('enable_custom_sources') and ( $update_source or !$uptodate );

    return $uptodate;
}

=pod

=head2 $cb-&gt;__check_uptodate( file =&gt; $file, name =&gt; $name, [update_source =&gt; BOOL, verbose =&gt; BOOL] )

C&lt;__check_uptodate&gt; checks if a given source file is still up-to-date
and if not, or when C&lt;update_source&gt; is true, will re-fetch the source
file.

Takes the following arguments:

=over 4

=item file

The source file to check.

=item name

The internal shortcut name for the source file (used for config
lookups).

=item update_source

Flag to force updating of sourcefiles regardless.

=item verbose

Boolean to indicate whether to be verbose or not.

=back

Returns a boolean value indicating whether the current files are up
to date or not.

=cut

### this method checks whether or not the source files we are using are still up to date
sub __check_uptodate {
    my $self = shift;
    my %hash = @_;
    my $conf = $self-&gt;configure_object;


    my $tmpl = {
        file            =&gt; { required =&gt; 1 },
        name            =&gt; { required =&gt; 1 },
        update_source   =&gt; { default =&gt; 0 },
        verbose         =&gt; { default =&gt; $conf-&gt;get_conf('verbose') },
    };

    my $args = check( $tmpl, \%hash ) or return;

    my $flag;
    unless ( -e $args-&gt;{'file'} &amp;&amp; (
            ( stat $args-&gt;{'file'} )[9]
            + $conf-&gt;_get_source('update') )
            &gt; time ) {
        $flag = 1;
    }

    if ( $flag or $args-&gt;{'update_source'} ) {

         if ( $self-&gt;_update_source( name =&gt; $args-&gt;{'name'} ) ) {
              return 0;       # return 0 so 'uptodate' will be set to 0, meaning no
                              # use of previously stored hashrefs!
         } else {
              msg( loc("Unable to update source, attempting to get away with using old source file!"), $args-&gt;{verbose} );
              return 1;
         }

    } else {
        return 1;
    }
}

=pod

=head2 $cb-&gt;_update_source( name =&gt; $name, [path =&gt; $path, verbose =&gt; BOOL] )

This method does the actual fetching of source files.

It takes the following arguments:

=over 4

=item name

The internal shortcut name for the source file (used for config
lookups).

=item path

The full path where to write the files.

=item verbose

Boolean to indicate whether to be verbose or not.

=back

Returns a boolean to indicate success.

=cut

### this sub fetches new source files ###
sub _update_source {
    my $self = shift;
    my %hash = @_;
    my $conf = $self-&gt;configure_object;

    my $verbose;
    my $tmpl = {
        name    =&gt; { required =&gt; 1 },
        path    =&gt; { default =&gt; $conf-&gt;get_conf('base') },
        verbose =&gt; { default =&gt; $conf-&gt;get_conf('verbose'), store =&gt; \$verbose },
    };

    my $args = check( $tmpl, \%hash ) or return;


    my $path = $args-&gt;{path};
    {   ### this could use a clean up - Kane
        ### no worries about the / -&gt; we get it from the _ftp configuration, so
        ### it's not platform dependant. -kane
        my ($dir, $file) = $conf-&gt;_get_mirror( $args-&gt;{'name'} ) =~ m|(.+/)(.+)$|sg;

        msg( loc("Updating source file '%1'", $file), $verbose );

        my $fake = CPANPLUS::Module::Fake-&gt;new(
                        module  =&gt; $args-&gt;{'name'},
                        path    =&gt; $dir,
                        package =&gt; $file,
                        _id     =&gt; $self-&gt;_id,
                    );

        ### can't use $fake-&gt;fetch here, since -&gt;parent won't work --
        ### the sources haven't been saved yet
        my $rv = $self-&gt;_fetch(
                    module      =&gt; $fake,
                    fetchdir    =&gt; $path,
                    force       =&gt; 1,
                );


        unless ($rv) {
            error( loc("Couldn't fetch '%1'", $file) );
            return;
        }

        $self-&gt;_update_timestamp( file =&gt; File::Spec-&gt;catfile($path, $file) );
    }

    return 1;
}

=pod

=head2 $cb-&gt;__create_author_tree([path =&gt; $path, uptodate =&gt; BOOL, verbose =&gt; BOOL])

This method opens a source files and parses its contents into a
searchable author-tree or restores a file-cached version of a
previous parse, if the sources are uptodate and the file-cache exists.

It takes the following arguments:

=over 4

=item uptodate

A flag indicating whether the file-cache is uptodate or not.

=item path

The absolute path to the directory holding the source files.

=item verbose

A boolean flag indicating whether or not to be verbose.

=back

Will get information from the config file by default.

Returns a tree on success, false on failure.

=cut

sub __create_author_tree {
    my $self = shift;
    my %hash = @_;
    my $conf = $self-&gt;configure_object;


    my $tmpl = {
        path     =&gt; { default =&gt; $conf-&gt;get_conf('base') },
        verbose  =&gt; { default =&gt; $conf-&gt;get_conf('verbose') },
        uptodate =&gt; { default =&gt; 0 },
    };

    my $args = check( $tmpl, \%hash ) or return;

    my $file = File::Spec-&gt;catfile(
                                $args-&gt;{path},
                                $conf-&gt;_get_source('auth')
                            );

    msg(loc("Rebuilding author tree, this might take a while"),
        $args-&gt;{verbose});

    ### extract the file ###
    my $ae      = Archive::Extract-&gt;new( archive =&gt; $file ) or return;
    my $out     = STRIP_GZ_SUFFIX-&gt;($file);

    ### make sure to set the PREFER_BIN flag if desired ###
    {   local $Archive::Extract::PREFER_BIN = $conf-&gt;get_conf('prefer_bin');
        $ae-&gt;extract( to =&gt; $out )                              or return;
    }

    my $cont    = $self-&gt;_get_file_contents( file =&gt; $out ) or return;

    ### don't need it anymore ###
    unlink $out;

    my ($tot,$prce,$prc,$idx);

    $args-&gt;{verbose}
   and local $|=1,
       $tot = scalar(split /\n/, $cont),
       ($prce, $prc, $idx) = (int $tot / 25, 0, 0);

    $args-&gt;{verbose}
   and print "\t0%";

    for ( split /\n/, $cont ) {
        my($id, $name, $email) = m/^alias \s+
                                    (\S+) \s+
                                    "\s* ([^\"\&lt;]+?) \s* &lt;(.+)&gt; \s*"
                                /x;

        $self-&gt;_add_author_object(
            author  =&gt; $name,           #authors name
            email   =&gt; $email,          #authors email address
            cpanid  =&gt; $id,             #authors CPAN ID
        ) or error( loc("Could not add author '%1'", $name ) );

   $args-&gt;{verbose}
       and (
          $idx++,

          ($idx==$prce
         and ($prc+=4,$idx=0,print ".")),

              (($prc % 10)
              or $idx
              or print $prc,'%')
      );

    }

    $args-&gt;{verbose}
   and print "\n";


    return $self-&gt;_atree;

} #__create_author_tree

=pod

=head2 $cb-&gt;_create_mod_tree([path =&gt; $path, uptodate =&gt; BOOL, verbose =&gt; BOOL])

This method opens a source files and parses its contents into a
searchable module-tree or restores a file-cached version of a
previous parse, if the sources are uptodate and the file-cache exists.

It takes the following arguments:

=over 4

=item uptodate

A flag indicating whether the file-cache is up-to-date or not.

=item path

The absolute path to the directory holding the source files.

=item verbose

A boolean flag indicating whether or not to be verbose.

=back

Will get information from the config file by default.

Returns a tree on success, false on failure.

=cut

### this builds a hash reference with the structure of the cpan module tree ###
sub _create_mod_tree {
    my $self = shift;
    my %hash = @_;
    my $conf = $self-&gt;configure_object;


    my $tmpl = {
        path     =&gt; { default =&gt; $conf-&gt;get_conf('base') },
        verbose  =&gt; { default =&gt; $conf-&gt;get_conf('verbose') },
        uptodate =&gt; { default =&gt; 0 },
    };

    my $args = check( $tmpl, \%hash ) or return undef;
    my $file = File::Spec-&gt;catfile($args-&gt;{path}, $conf-&gt;_get_source('mod'));

    msg(loc("Rebuilding module tree, this might take a while"),
        $args-&gt;{verbose});


    my $dslip_tree = $self-&gt;__create_dslip_tree( %$args );

    ### extract the file ###
    my $ae      = Archive::Extract-&gt;new( archive =&gt; $file ) or return;
    my $out     = STRIP_GZ_SUFFIX-&gt;($file);

    ### make sure to set the PREFER_BIN flag if desired ###
    {   local $Archive::Extract::PREFER_BIN = $conf-&gt;get_conf('prefer_bin');
        $ae-&gt;extract( to =&gt; $out )                              or return;
    }

    my $content = $self-&gt;_get_file_contents( file =&gt; $out ) or return;
    my $lines   = $content =~ tr/\n/\n/;

    ### don't need it anymore ###
    unlink $out;

    my($past_header, $count, $tot, $prce, $prc, $idx);

    $args-&gt;{verbose}
   and local $|=1,
       $tot = scalar(split /\n/, $content),
       ($prce, $prc, $idx) = (int $tot / 25, 0, 0);

    $args-&gt;{verbose}
   and print "\t0%";

    for ( split /\n/, $content ) {
        ### quick hack to read past the header of the file ###
        ### this is still rather evil... fix some time - Kane
        if( m|^\s*$| ) {
            unless( $count ) {
                error(loc("Could not determine line count from %1", $file));
                return;
            }
            $past_header = 1;
        }

        ### we're still in the header -- find the amount of lines we expect
        unless( $past_header ) {

            ### if the line count doesn't match what we expect, bail out
            ### this should address: #45644: detect broken index
            $count = $1 if /^Line-Count:\s+(\d+)/;
            if( $count ) {
                if( $lines &lt; $count ) {
                    error(loc("Expected to read at least %1 lines, but %2 ".
                              "contains only %3 lines!",
                              $count, $file, $lines ));
                    return;
                }
            }
            ### still in the header, keep moving
            next;
        }

        ### skip empty lines ###
        next unless /\S/;
        chomp;

        my @data = split /\s+/;

        ### filter out the author and filename as well ###
        ### authors can apparently have digits in their names,
        ### and dirs can have dots... blah!
        my ($author, $package) = $data[2] =~
                m|  (?:[A-Z\d-]/)?
                    (?:[A-Z\d-]{2}/)?
                    ([A-Z\d-]+) (?:/[\S]+)?/
                    ([^/]+)$
                |xsg;

        ### remove file name from the path
        $data[2] =~ s|/[^/]+$||;

        my $aobj = $self-&gt;author_tree($author);
        unless( $aobj ) {
            error( loc( "No such author '%1' -- can't make module object " .
                        "'%2' that is supposed to belong to this author",
                        $author, $data[0] ) );
            next;
        }

        ### adding the dslip info
        ### probably can use some optimization
        my $dslip;
        for my $item ( qw[ statd stats statl stati statp ] ) {
            ### checking if there's an entry in the dslip info before
            ### catting it on. appeasing warnings this way
            $dslip .=   $dslip_tree-&gt;{ $data[0] }-&gt;{$item}
                            ? $dslip_tree-&gt;{ $data[0] }-&gt;{$item}
                            : ' ';
        }

        ### XXX this could be sped up if we used author names, not author
        ### objects in creation, and then look them up in the author tree
        ### when needed. This will need a fix to all the places that create
        ### fake author/module objects as well.

        ### callback to store the individual object
        $self-&gt;_add_module_object(
            module      =&gt; $data[0],            # full module name
            version     =&gt; ($data[1] eq 'undef' # version number
                                ? '0.0'
                                : $data[1]),
            path        =&gt; File::Spec::Unix-&gt;catfile(
                                $conf-&gt;_get_mirror('base'),
                                $data[2],
                            ),          # extended path on the cpan mirror,
                                        # like /A/AB/ABIGAIL
            comment     =&gt; $data[3],    # comment on the module
            author      =&gt; $aobj,
            package     =&gt; $package,    # package name, like
                                        # 'foo-bar-baz-1.03.tar.gz'
            description =&gt; $dslip_tree-&gt;{ $data[0] }-&gt;{'description'},
            dslip       =&gt; $dslip,
            mtime       =&gt; '',
        ) or error( loc( "Could not add module '%1'", $data[0] ) );

   $args-&gt;{verbose}
       and (
          $idx++,

          ($idx==$prce
         and ($prc+=4,$idx=0,print ".")),

              (($prc % 10)
              or $idx
              or print $prc,'%')
      );

    } #for

    $args-&gt;{verbose}
   and print "\n";

    return $self-&gt;_mtree;

} #_create_mod_tree

=pod

=head2 $cb-&gt;__create_dslip_tree([path =&gt; $path, uptodate =&gt; BOOL, verbose =&gt; BOOL])

This method opens a source files and parses its contents into a
searchable dslip-tree or restores a file-cached version of a
previous parse, if the sources are uptodate and the file-cache exists.

It takes the following arguments:

=over 4

=item uptodate

A flag indicating whether the file-cache is uptodate or not.

=item path

The absolute path to the directory holding the source files.

=item verbose

A boolean flag indicating whether or not to be verbose.

=back

Will get information from the config file by default.

Returns a tree on success, false on failure.

=cut

sub __create_dslip_tree {
    my $self = shift;
    my %hash = @_;
    my $conf = $self-&gt;configure_object;

    my $tmpl = {
        path     =&gt; { default =&gt; $conf-&gt;get_conf('base') },
        verbose  =&gt; { default =&gt; $conf-&gt;get_conf('verbose') },
        uptodate =&gt; { default =&gt; 0 },
    };

    my $args = check( $tmpl, \%hash ) or return;

    ### get the file name of the source ###
    my $file = File::Spec-&gt;catfile($args-&gt;{path}, $conf-&gt;_get_source('dslip'));

    ### extract the file ###
    my $ae      = Archive::Extract-&gt;new( archive =&gt; $file ) or return;
    my $out     = STRIP_GZ_SUFFIX-&gt;($file);

    ### make sure to set the PREFER_BIN flag if desired ###
    {   local $Archive::Extract::PREFER_BIN = $conf-&gt;get_conf('prefer_bin');
        $ae-&gt;extract( to =&gt; $out )                              or return;
    }

    my $in      = $self-&gt;_get_file_contents( file =&gt; $out ) or return;

    ### don't need it anymore ###
    unlink $out;


    ### get rid of the comments and the code ###
    ### need a smarter parser, some people have this in their dslip info:
    # [
    # 'Statistics::LTU',
    # 'R',
    # 'd',
    # 'p',
    # 'O',
    # '?',
    # 'Implements Linear Threshold Units',
    # ...skipping...
    # "\x{c4}dd \x{fc}ml\x{e4}\x{fc}ts t\x{f6} \x{eb}v\x{eb}r\x{ff}th\x{ef}ng!",
    # 'BENNIE',
    # '11'
    # ],
    ### also, older versions say:
    ### $cols = [....]
    ### and newer versions say:
    ### $CPANPLUS::Modulelist::cols = [...]
    ### split '$cols' and '$data' into 2 variables ###
    ### use this regex to make sure dslips with ';' in them don't cause
    ### parser errors
    my ($ds_one, $ds_two) = ($in =~ m|.+}\s+
                              (\$(?:CPAN::Modulelist::)?cols.*?)
                              (\$(?:CPAN::Modulelist::)?data.*)
                           |sx);

    ### eval them into existence ###
    ### still not too fond of this solution - kane ###
    my ($cols, $data);
    {   #local $@; can't use this, it's buggy -kane

        $cols = eval $ds_one;
        error( loc("Error in eval of dslip source files: %1", $@) ) if $@;

        $data = eval $ds_two;
        error( loc("Error in eval of dslip source files: %1", $@) ) if $@;

    }

    my $tree = {};
    my $primary = "modid";

    ### this comes from CPAN::Modulelist
    ### which is in 03modlist.data.gz
    for (@$data){
        my %hash;
        @hash{@$cols} = @$_;
        $tree-&gt;{$hash{$primary}} = \%hash;
    }

    return $tree;

} #__create_dslip_tree

=pod

=head2 $cb-&gt;_dslip_defs ()

This function returns the definition structure (ARRAYREF) of the
dslip tree.

=cut

### these are the definitions used for dslip info
### they shouldn't change over time.. so hardcoding them doesn't appear to
### be a problem. if it is, we need to parse 03modlist.data better to filter
### all this out.
### right now, this is just used to look up dslip info from a module
sub _dslip_defs {
    my $self = shift;

    my $aref = [

        # D
        [ q|Development Stage|, {
            i   =&gt; loc('Idea, listed to gain consensus or as a placeholder'),
            c   =&gt; loc('under construction but pre-alpha (not yet released)'),
            a   =&gt; loc('Alpha testing'),
            b   =&gt; loc('Beta testing'),
            R   =&gt; loc('Released'),
            M   =&gt; loc('Mature (no rigorous definition)'),
            S   =&gt; loc('Standard, supplied with Perl 5'),
        }],

        # S
        [ q|Support Level|, {
            m   =&gt; loc('Mailing-list'),
            d   =&gt; loc('Developer'),
            u   =&gt; loc('Usenet newsgroup comp.lang.perl.modules'),
            n   =&gt; loc('None known, try comp.lang.perl.modules'),
            a   =&gt; loc('Abandoned; volunteers welcome to take over maintenance'),
        }],

        # L
        [ q|Language Used|, {
            p   =&gt; loc('Perl-only, no compiler needed, should be platform independent'),
            c   =&gt; loc('C and perl, a C compiler will be needed'),
            h   =&gt; loc('Hybrid, written in perl with optional C code, no compiler needed'),
            '+' =&gt; loc('C++ and perl, a C++ compiler will be needed'),
            o   =&gt; loc('perl and another language other than C or C++'),
        }],

        # I
        [ q|Interface Style|, {
            f   =&gt; loc('plain Functions, no references used'),
            h   =&gt; loc('hybrid, object and function interfaces available'),
            n   =&gt; loc('no interface at all (huh?)'),
            r   =&gt; loc('some use of unblessed References or ties'),
            O   =&gt; loc('Object oriented using blessed references and/or inheritance'),
        }],

        # P
        [ q|Public License|, {
            p   =&gt; loc('Standard-Perl: user may choose between GPL and Artistic'),
            g   =&gt; loc('GPL: GNU General Public License'),
            l   =&gt; loc('LGPL: "GNU Lesser General Public License" (previously known as "GNU Library General Public License")'),
            b   =&gt; loc('BSD: The BSD License'),
            a   =&gt; loc('Artistic license alone'),
            o   =&gt; loc('other (but distribution allowed without restrictions)'),
        }],
    ];

    return $aref;
}

=head2 $file = $cb-&gt;_add_custom_module_source( uri =&gt; URI, [verbose =&gt; BOOL] );

Adds a custom source index and updates it based on the provided URI.

Returns the full path to the index file on success or false on failure.

=cut

sub _add_custom_module_source {
    my $self = shift;
    my $conf = $self-&gt;configure_object;
    my %hash = @_;

    my($verbose,$uri);
    my $tmpl = {
        verbose =&gt; { default =&gt; $conf-&gt;get_conf('verbose'),
                     store   =&gt; \$verbose },
        uri     =&gt; { required =&gt; 1, store =&gt; \$uri }
    };

    check( $tmpl, \%hash ) or return;

    ### what index file should we use on disk?
    my $index = $self-&gt;__custom_module_source_index_file( uri =&gt; $uri );

    ### already have it.
    if( IS_FILE-&gt;( $index ) ) {
        msg(loc("Source '%1' already added", $uri));
        return 1;
    }

    ### do we need to create the targe dir?
    {   my $dir = dirname( $index );
        unless( IS_DIR-&gt;( $dir ) ) {
            $self-&gt;_mkdir( dir =&gt; $dir ) or return
        }
    }

    ### write the file
    my $fh = OPEN_FILE-&gt;( $index =&gt; '&gt;' ) or do {
        error(loc("Could not open index file for '%1'", $uri));
        return;
    };

    ### basically we 'touched' it. Check the return value, may be
    ### important on win32 and similar OS, where there's file length
    ### limits
    close $fh or do {
        error(loc("Could not write index file to disk for '%1'", $uri));
        return;
    };

    $self-&gt;__update_custom_module_source(
                remote  =&gt; $uri,
                local   =&gt; $index,
                verbose =&gt; $verbose,
            ) or do {
                ### we faild to update it, we probably have an empty
                ### possibly silly filename on disk now -- remove it
                1 while unlink $index;
                return;
            };

    return $index;
}

=head2 $index = $cb-&gt;__custom_module_source_index_file( uri =&gt; $uri );

Returns the full path to the encoded index file for C&lt;$uri&gt;, as used by
all C&lt;custom module source&gt; routines.

=cut

sub __custom_module_source_index_file {
    my $self = shift;
    my $conf = $self-&gt;configure_object;
    my %hash = @_;

    my($verbose,$uri);
    my $tmpl = {
        uri     =&gt; { required =&gt; 1, store =&gt; \$uri }
    };

    check( $tmpl, \%hash ) or return;

    my $index = File::Spec-&gt;catfile(
                    $conf-&gt;get_conf('base'),
                    $conf-&gt;_get_build('custom_sources'),
                    $self-&gt;_uri_encode( uri =&gt; $uri ),
                );

    return $index;
}

=head2 $file = $cb-&gt;_remove_custom_module_source( uri =&gt; URI, [verbose =&gt; BOOL] );

Removes a custom index file based on the URI provided.

Returns the full path to the index file on success or false on failure.

=cut

sub _remove_custom_module_source {
    my $self = shift;
    my $conf = $self-&gt;configure_object;
    my %hash = @_;

    my($verbose,$uri);
    my $tmpl = {
        verbose =&gt; { default =&gt; $conf-&gt;get_conf('verbose'),
                     store   =&gt; \$verbose },
        uri     =&gt; { required =&gt; 1, store =&gt; \$uri }
    };

    check( $tmpl, \%hash ) or return;

    ### use uri =&gt; local, instead of the other way around
    my %files = reverse $self-&gt;__list_custom_module_sources;

    ### On VMS the case of key to %files can be either exact or lower case
    ### XXX abstract this lookup out? --kane
    my $file = $files{ $uri };
    $file    = $files{ lc $uri } if !defined($file) &amp;&amp; ON_VMS;

    unless (defined $file) {
        error(loc("No such custom source '%1'", $uri));
        return;
    };

    1 while unlink $file;

    if( IS_FILE-&gt;( $file ) ) {
        error(loc("Could not remove index file '%1' for custom source '%2'",
                    $file, $uri));
        return;
    }

    msg(loc("Successfully removed index file for '%1'", $uri), $verbose);

    return $file;
}

=head2 %files = $cb-&gt;__list_custom_module_sources

This method scans the 'custom-sources' directory in your base directory
for additional sources to include in your module tree.

Returns a list of key value pairs as follows:

  /full/path/to/source/file%3Fencoded =&gt; http://decoded/mirror/path

=cut

sub __list_custom_module_sources {
    my $self = shift;
    my $conf = $self-&gt;configure_object;

    my($verbose);
    my $tmpl = {
        verbose =&gt; { default =&gt; $conf-&gt;get_conf('verbose'),
                     store   =&gt; \$verbose },
    };

    my $dir = File::Spec-&gt;catdir(
                    $conf-&gt;get_conf('base'),
                    $conf-&gt;_get_build('custom_sources'),
                );

    unless( IS_DIR-&gt;( $dir ) ) {
        msg(loc("No '%1' dir, skipping custom sources", $dir), $verbose);
        return;
    }

    ### unencode the files
    ### skip ones starting with # though
    my %files = map {
        my $org = $_;
        my $dec = $self-&gt;_uri_decode( uri =&gt; $_ );
        File::Spec-&gt;catfile( $dir, $org ) =&gt; $dec
    } grep { $_ !~ /^#/ } READ_DIR-&gt;( $dir );

    return %files;
}

=head2 $bool = $cb-&gt;__update_custom_module_sources( [verbose =&gt; BOOL] );

Attempts to update all the index files to your custom module sources.

If the index is missing, and it's a C&lt;file://&gt; uri, it will generate
a new local index for you.

Return true on success, false on failure.

=cut

sub __update_custom_module_sources {
    my $self = shift;
    my $conf = $self-&gt;configure_object;
    my %hash = @_;

    my $verbose;
    my $tmpl = {
        verbose =&gt; { default =&gt; $conf-&gt;get_conf('verbose'),
                     store   =&gt; \$verbose }
    };

    check( $tmpl, \%hash ) or return;

    my %files = $self-&gt;__list_custom_module_sources;

    ### uptodate check has been done a few levels up.
    my $fail;
    while( my($local,$remote) = each %files ) {

        $self-&gt;__update_custom_module_source(
                    remote  =&gt; $remote,
                    local   =&gt; $local,
                    verbose =&gt; $verbose,
                ) or ( $fail++, next );
    }

    error(loc("Failed updating one or more remote sources files")) if $fail;

    return if $fail;
    return 1;
}

=head2 $ok = $cb-&gt;__update_custom_module_source

Attempts to update all the index files to your custom module sources.

If the index is missing, and it's a C&lt;file://&gt; uri, it will generate
a new local index for you.

Return true on success, false on failure.

=cut

sub __update_custom_module_source {
    my $self = shift;
    my $conf = $self-&gt;configure_object;
    my %hash = @_;

    my($verbose,$local,$remote);
    my $tmpl = {
        verbose =&gt; { default  =&gt; $conf-&gt;get_conf('verbose'),
                     store    =&gt; \$verbose },
        local   =&gt; { store    =&gt; \$local, allow =&gt; FILE_EXISTS },
        remote  =&gt; { required =&gt; 1, store =&gt; \$remote },
    };

    check( $tmpl, \%hash ) or return;

    msg( loc("Updating sources from '%1'", $remote), $verbose);

    ### if you didn't provide a local file, we'll look in your custom
    ### dir to find the local encoded version for you
    $local ||= do {
        ### find all files we know of
        my %files = reverse $self-&gt;__list_custom_module_sources or do {
            error(loc("No custom modules sources defined -- need '%1' argument",
                      'local'));
            return;
        };

        ### On VMS the case of key to %files can be either exact or lower case
        ### XXX abstract this lookup out? --kane
        my $file = $files{ $remote };
        $file    = $files{ lc $remote } if !defined ($file) &amp;&amp; ON_VMS;

        ### return the local file we're supposed to use
        $file or do {
            error(loc("Remote source '%1' unknown -- needs '%2' argument",
                      $remote, 'local'));
            return;
        };
    };

    my $uri =  join '/', $remote, $conf-&gt;_get_source('custom_index');
    my $ff  =  File::Fetch-&gt;new( uri =&gt; $uri );

    ### tempdir doesn't clean up by default, as opposed to tempfile()
    ### so add it explicitly.
    my $dir =  tempdir( CLEANUP =&gt; 1 );

    my $res =  do {
                    local $File::Fetch::WARN = 0;
                    local $File::Fetch::TIMEOUT = $conf-&gt;get_conf('timeout');
                    $ff-&gt;fetch( to =&gt; $dir );
               };

    ### couldn't get the file
    unless( $res ) {

        ### it's not a local scheme, so can't auto index
        unless( $ff-&gt;scheme eq 'file' ) {
            error(loc("Could not update sources from '%1': %2",
                      $remote, $ff-&gt;error ));
            return;

        ### it's a local uri, we can index it ourselves
        } else {
            msg(loc("No index file found at '%1', generating one",
                    $ff-&gt;uri), $verbose );

            ### ON VMS, if you are working with a UNIX file specification,
            ### you need currently use the UNIX variants of the File::Spec.
            my $ff_path = do {
                my $file_class = 'File::Spec';
                $file_class .= '::Unix' if ON_VMS;
                $file_class-&gt;catdir( File::Spec::Unix-&gt;splitdir( $ff-&gt;path ) );
            };

            $self-&gt;__write_custom_module_index(
                path    =&gt; $ff_path,
                to      =&gt; $local,
                verbose =&gt; $verbose,
            ) or return;

            ### XXX don't write that here, __write_custom_module_index
            ### already prints this out
            #msg(loc("Index file written to '%1'", $to), $verbose);
        }

    ### copy it to the real spot and update its timestamp
    } else {
        $self-&gt;_move( file =&gt; $res, to =&gt; $local ) or return;
        $self-&gt;_update_timestamp( file =&gt; $local );

        msg(loc("Index file saved to '%1'", $local), $verbose);
    }

    return $local;
}

=head2 $bool = $cb-&gt;__write_custom_module_index( path =&gt; /path/to/packages, [to =&gt; /path/to/index/file, verbose =&gt; BOOL] )

Scans the C&lt;path&gt; you provided for packages and writes an index with all
the available packages to C&lt;$path/packages.txt&gt;. If you'd like the index
to be written to a different file, provide the C&lt;to&gt; argument.

Returns true on success and false on failure.

=cut

sub __write_custom_module_index {
    my $self = shift;
    my $conf = $self-&gt;configure_object;
    my %hash = @_;

    my ($verbose, $path, $to);
    my $tmpl = {
        verbose =&gt; { default =&gt; $conf-&gt;get_conf('verbose'),
                     store   =&gt; \$verbose },
        path    =&gt; { required =&gt; 1, allow =&gt; DIR_EXISTS, store =&gt; \$path },
        to      =&gt; { store =&gt; \$to },
    };

    check( $tmpl, \%hash ) or return;

    ### no explicit to? then we'll use our default
    $to ||= File::Spec-&gt;catfile( $path, $conf-&gt;_get_source('custom_index') );

    my @files;
    require File::Find;
    File::Find::find( sub {
        ### let's see if A::E can even parse it
        my $ae = do {
            local $Archive::Extract::WARN = 0;
            local $Archive::Extract::WARN = 0;
            Archive::Extract-&gt;new( archive =&gt; $File::Find::name )
        } or return;

        ### it's a type A::E recognize, so we can add it
        $ae-&gt;type or return;

        ### neither $_ nor $File::Find::name have the chunk of the path in
        ### it starting $path -- it's either only the filename, or the full
        ### path, so we have to strip it ourselves
        ### make sure to remove the leading slash as well.
        my $copy = $File::Find::name;
        my $re   = quotemeta($path);
        $copy    =~ s|^$re[\\/]?||i;

        push @files, $copy;

    }, $path );

    ### does the dir exist? if not, create it.
    {   my $dir = dirname( $to );
        unless( IS_DIR-&gt;( $dir ) ) {
            $self-&gt;_mkdir( dir =&gt; $dir ) or return
        }
    }

    ### create the index file
    my $fh = OPEN_FILE-&gt;( $to =&gt; '&gt;' ) or return;

    print $fh "$_\n" for @files;
    close $fh;

    msg(loc("Successfully written index file to '%1'", $to), $verbose);

    return $to;
}


=head2 $bool = $cb-&gt;__create_custom_module_entries( [verbose =&gt; BOOL] )

Creates entries in the module tree based upon the files as returned
by C&lt;__list_custom_module_sources&gt;.

Returns true on success, false on failure.

=cut

### use $auth_obj as a persistent version, so we don't have to recreate
### modules all the time
{   my $auth_obj;

    sub __create_custom_module_entries {
        my $self    = shift;
        my $conf    = $self-&gt;configure_object;
        my %hash    = @_;

        my $verbose;
        my $tmpl = {
            verbose     =&gt; { default =&gt; $conf-&gt;get_conf('verbose'), store =&gt; \$verbose },
        };

        check( $tmpl, \%hash ) or return undef;

        my %files = $self-&gt;__list_custom_module_sources;

        while( my($file,$name) = each %files ) {

            msg(loc("Adding packages from custom source '%1'", $name), $verbose);

            my $fh = OPEN_FILE-&gt;( $file ) or next;

            while( local $_ = &lt;$fh&gt; ) {
                chomp;
                next if /^#/;
                next unless /\S+/;

                ### join on / -- it's a URI after all!
                my $parse = join '/', $name, $_;

                ### try to make a module object out of it
                my $mod = $self-&gt;parse_module( module =&gt; $parse ) or (
                    error(loc("Could not parse '%1'", $_)),
                    next
                );

                ### mark this object with a custom author
                $auth_obj ||= do {
                    my $id = CUSTOM_AUTHOR_ID;

                    ### if the object is being created for the first time,
                    ### make sure there's an entry in the author tree as
                    ### well, so we can search on the CPAN ID
                    $self-&gt;author_tree-&gt;{ $id } =
                        CPANPLUS::Module::Author::Fake-&gt;new( cpanid =&gt; $id );
                };

                $mod-&gt;author( $auth_obj );

                ### and now add it to the module tree -- this MAY
                ### override things of course
                if( my $old_mod = $self-&gt;module_tree( $mod-&gt;module ) ) {

                    ### On VMS use the old module name to get the real case
                    $mod-&gt;module( $old_mod-&gt;module ) if ON_VMS;

                    msg(loc("About to overwrite module tree entry for '%1' with '%2'",
                            $mod-&gt;module, $mod-&gt;package), $verbose);
                }

                ### mark where it came from
                $mod-&gt;description( loc("Custom source from '%1'",$name) );

                ### store it in the module tree
                $self-&gt;module_tree-&gt;{ $mod-&gt;module } = $mod;
            }
        }

        return 1;
    }
}

1;
</pre></body></html>