/[cvs]/nfo/perl/libs/Data/Storage.pm
ViewVC logotype

Diff of /nfo/perl/libs/Data/Storage.pm

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 1.5 by joko, Tue Oct 29 19:24:18 2002 UTC revision 1.12 by joko, Thu Dec 12 02:50:15 2002 UTC
# Line 4  Line 4 
4  #  #
5  # See COPYRIGHT section in pod text below for usage and distribution rights.  # See COPYRIGHT section in pod text below for usage and distribution rights.
6  #  #
7  #################################  ############################################
8  #  #
9  #  $Log$  #  $Log$
10    #  Revision 1.12  2002/12/12 02:50:15  joko
11    #  + this now (unfortunately) needs DBI for some helper functions
12    #  + TODO: these have to be refactored to another scope! (soon!)
13    #
14    #  Revision 1.11  2002/12/11 06:53:19  joko
15    #  + updated pod
16    #
17    #  Revision 1.10  2002/12/07 03:37:23  joko
18    #  + updated pod
19    #
20    #  Revision 1.9  2002/12/01 22:15:45  joko
21    #  - sub createDb: moved to handler
22    #
23    #  Revision 1.8  2002/11/29 04:48:23  joko
24    #  + updated pod
25    #
26    #  Revision 1.7  2002/11/17 06:07:18  joko
27    #  + creating the handler is easier than proposed first - for now :-)
28    #  + sub testAvailability
29    #
30    #  Revision 1.6  2002/11/09 01:04:58  joko
31    #  + updated pod
32    #
33  #  Revision 1.5  2002/10/29 19:24:18  joko  #  Revision 1.5  2002/10/29 19:24:18  joko
34  #  - reduced logging  #  - reduced logging
35  #  + added some pod  #  + added some pod
# Line 27  Line 50 
50  #  Revision 1.1  2002/10/10 03:43:12  cvsjoko  #  Revision 1.1  2002/10/10 03:43:12  cvsjoko
51  #  + new  #  + new
52  #  #
53  #################################  ############################################
54    
 # aim_V1: should encapsulate Tangram, DBI, DBD::CSV and LWP:: to access them in an unordinary way ;)  
 # aim_V2: introduce a generic layered structure, refactor *SUBLAYER*-stuff, make (e.g.) this possible:  
 #               - Perl Data::Storage[DBD::CSV]  ->  Perl LWP::  ->  Internet HTTP/FTP/*  ->  Host Daemon  ->  csv-file  
55    
56  BEGIN {  BEGIN {
57  $Data::Storage::VERSION = 0.01;    $Data::Storage::VERSION = 0.02;
58  }  }
59    
60    
61  =head1 NAME  =head1 NAME
62    
63  Data::Storage - Interface for accessing various Storage implementations for Perl in an independent way    Data::Storage - Interface for accessing various Storage implementations for Perl in an independent way
64    
65    
66    =head1 AIMS
67    
68      - should encapsulate Tangram, DBI, DBD::CSV and LWP:: to access them in an unordinary (more convenient) way ;)
69      - introduce a generic layered structure, refactor *SUBLAYER*-stuff, make (e.g.) this possible:
70        Perl Data::Storage[DBD::CSV]  ->  Perl LWP::  ->  Internet HTTP/FTP/*  ->  Host Daemon  ->  csv-file
71      - provide generic synchronization mechanisms across arbitrary/multiple storages based on ident/checksum
72        maybe it's possible to have schema-, structural- and semantical modifications synchronized???
73    
74    
75  =head1 SYNOPSIS  =head1 SYNOPSIS
76    
77    ... the basic way:  =head2 BASIC ACCESS
78    
79    =head2 ADVANCED ACCESS
80    
81    ... via inheritance:    ... via inheritance:
82        
# Line 63  Data::Storage - Interface for accessing Line 94  Data::Storage - Interface for accessing
94      $self->{storage}->insert($proxyObj);      $self->{storage}->insert($proxyObj);
95    
96    
97    =head2 SYNCHRONIZATION
98    
99      my $nodemapping = {
100        'LangText' => 'langtexts.csv',
101        'Currency' => 'currencies.csv',
102        'Country'  => 'countries.csv',
103      };
104    
105      my $propmapping = {
106        'LangText' => [
107          [ 'source:lcountrykey'  =>  'target:country' ],
108          [ 'source:lkey'         =>  'target:key' ],
109          [ 'source:lvalue'       =>  'target:text' ],
110        ],
111        'Currency' => [
112          [ 'source:ckey'         =>  'target:key' ],
113          [ 'source:cname'        =>  'target:text' ],
114        ],
115        'Country' => [
116          [ 'source:ckey'         =>  'target:key' ],
117          [ 'source:cname'        =>  'target:text' ],
118        ],
119      };
120    
121      sub syncResource {
122    
123        my $self = shift;
124        my $node_source = shift;
125        my $mode = shift;
126        my $opts = shift;
127        
128        $mode ||= '';
129        $opts->{erase} ||= 0;
130        
131        $logger->info( __PACKAGE__ . "->syncResource( node_source $node_source mode $mode erase $opts->{erase} )");
132      
133        # resolve metadata for syncing requested resource
134        my $node_target = $nodemapping->{$node_source};
135        my $mapping = $propmapping->{$node_source};
136        
137        if (!$node_target || !$mapping) {
138          # loggger.... "no target, sorry!"
139          print "error while resolving resource metadata", "\n";
140          return;
141        }
142        
143        if ($opts->{erase}) {
144          $self->_erase_all($node_source);
145        }
146      
147        # create new sync object
148        my $sync = Data::Transfer::Sync->new(
149          storages => {
150            L => $self->{bizWorks}->{backend},
151            R => $self->{bizWorks}->{resources},
152          },
153          id_authorities        =>  [qw( L ) ],
154          checksum_authorities  =>  [qw( L ) ],
155          write_protected       =>  [qw( R ) ],
156          verbose               =>  1,
157        );
158        
159        # sync
160        # todo: filter!?
161        $sync->syncNodes( {
162          direction       =>  $mode,                 # | +PUSH | +PULL | -FULL | +IMPORT | -EXPORT
163          method          =>  'checksum',            # | -timestamp | -manual
164          source          =>  "L:$node_source",
165          source_ident    =>  'storage_method:id',
166          source_exclude  =>  [qw( id cs )],
167          target          =>  "R:$node_target",
168          target_ident    =>  'property:oid',
169          mapping         =>  $mapping,
170        } );
171    
172      }
173    
174    
175  =head2 NOTE  =head2 NOTE
176    
177  This module heavily relies on DBI and Tangram, but adds a lot of additional bugs and quirks.    This module heavily relies on DBI and Tangram, but adds a lot of additional bugs and quirks.
178  Please look at their documentation and this code for additional information.    Please look at their documentation and/or this code for additional information.
179    
180    
181    =head1 REQUIREMENTS
182    
183      For full functionality:
184        DBI              from CPAN
185        DBD::mysql       from CPAN
186        Tangram 2.04     from CPAN         (hmmm, 2.04 won't do in some cases)
187        Tangram 2.05     from http://...   (2.05 seems okay but there are also additional patches from our side)
188        Class::Tangram   from CPAN
189        DBD::CSV         from CPAN
190        MySQL::Diff      from http://adamspiers.org/computing/mysqldiff/
191        ... and all their dependencies
192    
193  =cut  =cut
194    
# Line 80  use strict; Line 201  use strict;
201  use warnings;  use warnings;
202    
203  use Data::Storage::Locator;  use Data::Storage::Locator;
204    use Data::Dumper;
205    
206    # TODO: wipe out!
207    use DBI;
208    
209    # TODO: actually implement level (integrate with Log::Dispatch)
210  my $TRACELEVEL = 0;  my $TRACELEVEL = 0;
211    
212  # get logger instance  # get logger instance
# Line 90  sub new { Line 216  sub new {
216    my $invocant = shift;    my $invocant = shift;
217    my $class = ref($invocant) || $invocant;    my $class = ref($invocant) || $invocant;
218    #my @args = normalizeArgs(@_);    #my @args = normalizeArgs(@_);
219      
220    my $arg_locator = shift;    my $arg_locator = shift;
221    my $arg_options = shift;    my $arg_options = shift;
222      
223    #my $self = { STORAGEHANDLE => undef, @_ };    #my $self = { STORAGEHANDLE => undef, @_ };
224    my $self = { STORAGEHANDLE => undef, locator => $arg_locator, options => $arg_options };    my $self = { STORAGEHANDLE => undef, locator => $arg_locator, options => $arg_options };
225    $logger->debug( __PACKAGE__ . "[$self->{locator}->{type}]" . "->new(@_)" );    #$logger->debug( __PACKAGE__ . "[$self->{locator}->{type}]" . "->new(@_)" );
226      $logger->debug( __PACKAGE__ . "[$arg_locator->{type}]" . "->new(@_)" );
227    return bless $self, $class;    return bless $self, $class;
228  }  }
229    
# Line 107  sub AUTOLOAD { Line 234  sub AUTOLOAD {
234    #     - Deep recursion on subroutine "Data::Storage::AUTOLOAD"    #     - Deep recursion on subroutine "Data::Storage::AUTOLOAD"
235    #     - Deep recursion on subroutine "Data::Storage::Handler::Abstract::AUTOLOAD"    #     - Deep recursion on subroutine "Data::Storage::Handler::Abstract::AUTOLOAD"
236    #     - Deep recursion on anonymous subroutine at [...]    #     - Deep recursion on anonymous subroutine at [...]
237    # we also might filter log messages caused by logging itself in "advanced logging of AUTOLOAD calls"    # we also might filter log messages caused by logging to itself in "advanced logging of AUTOLOAD calls"
238        
239    my $self = shift;    my $self = shift;
240    our $AUTOLOAD;    our $AUTOLOAD;
# Line 129  sub AUTOLOAD { Line 256  sub AUTOLOAD {
256        $logstring .= "\t" x $tabcount . "(AUTOLOAD)";        $logstring .= "\t" x $tabcount . "(AUTOLOAD)";
257        # TODO: only ok if logstring doesn't contain        # TODO: only ok if logstring doesn't contain
258        #            e.g. "Data::Storage[Tangram]->insert(SystemEvent=HASH(0x5c0034c))          (AUTOLOAD)"        #            e.g. "Data::Storage[Tangram]->insert(SystemEvent=HASH(0x5c0034c))          (AUTOLOAD)"
259        # but that would be way too specific as long as we don't have an abstract handler for this  ;)        # but that would be _way_ too specific as long as we don't have an abstract handler for this  ;)
260        $logger->debug( $logstring );        $logger->debug( $logstring );
261          #print join('; ', @_);
262      }      }
263            
264    # filtering AUTOLOAD calls    # filtering AUTOLOAD calls and first-time-touch of the actual storage impl
265    if ($self->_filter_AUTOLOAD($method)) {    if ($self->_filter_AUTOLOAD($method)) {
266        #print "_accessStorage\n";
267      $self->_accessStorage();      $self->_accessStorage();
268      $self->{STORAGEHANDLE}->$method(@_);      $self->{STORAGEHANDLE}->$method(@_);
269    }    }
# Line 175  sub _accessStorage { Line 304  sub _accessStorage {
304    
305  sub _createStorageHandle {  sub _createStorageHandle {
306    my $self = shift;    my $self = shift;
   
307    my $type = $self->{locator}->{type};    my $type = $self->{locator}->{type};
308    $logger->debug( __PACKAGE__ .  "[$type]" . "->_createStorageHandle()" );    $logger->debug( __PACKAGE__ .  "[$type]" . "->_createStorageHandle()" );
309    
310    my $pkg = "Data::Storage::Handler::" . $type . "";    my $pkg = "Data::Storage::Handler::" . $type . "";
311        
312    # propagate args to handler    # try to load perl module at runtime
313    # needs some more thoughts! (not only "dbi" to Tangram, when (in future) db is not more the common case)    my $evalstr = "use $pkg;";
314    if ($type eq 'DBI') {    eval($evalstr);
315      use Data::Storage::Handler::DBI;    if ($@) {
316      #my @args = %{$self->{locator}->{dbi}};      $logger->error( __PACKAGE__ .  "[$type]" . "->_createStorageHandle(): $@" );
317      my @args = %{$self->{locator}};      return;
     # create new storage handle  
     $self->{STORAGEHANDLE} = $pkg->new( @args );  
   }  
   if ($type eq 'Tangram') {  
     use Data::Storage::Handler::Tangram;  
     #$self->{STORAGEHANDLE} = $pkg->new( dsn => $self->{locator}->{dbi}->{dsn} );  
     #my @args = %{$self->{locator}->{dbi}};  
     my @args = %{$self->{locator}};  
     # create new storage handle  
     $self->{STORAGEHANDLE} = $pkg->new( @args );  
   
     #$self->{STORAGEHANDLE_UNDERLYING} = $self->{STORAGEHANDLE}->getUnderlyingStorage();  
     #$self->{STORAGEHANDLE_UNDERLYING}->_configureCOREHANDLE();  
318    }    }
319        
320      # build up some additional arguments to pass on
321      #my @args = %{$self->{locator}};
322      my @args = ();
323    
324      # - create new storage handle object
325      # - propagate arguments to handler
326      # - pass locator by reference to be able to store status- or meta-information in it
327      $self->{STORAGEHANDLE} = $pkg->new( locator => $self->{locator}, @args );
328    
329  }  }
330    
331  sub addLogDispatchHandler {  sub addLogDispatchHandler {
# Line 233  sub addLogDispatchHandler { Line 357  sub addLogDispatchHandler {
357  }  }
358    
359  sub removeLogDispatchHandler {  sub removeLogDispatchHandler {
360      my $self = shift;
361        my $self = shift;    my $name = shift;
362        my $name = shift;    #my $logger = shift;
363        #my $logger = shift;    $logger->remove($name);
   
       $logger->remove($name);  
   
364  }  }
365    
366  sub getDbName {  sub getDbName {
# Line 250  sub getDbName { Line 371  sub getDbName {
371    return $database_name;    return $database_name;
372  }  }
373    
374    sub testAvailability {
375      my $self = shift;
376      my $status = $self->testDsn();
377      $self->{locator}->{status}->{available} = $status;
378      return $status;
379    }
380    
381    sub isConnected {
382      my $self = shift;
383      # TODO: REVIEW!
384      return 1 if $self->{STORAGEHANDLE};
385    }
386    
387  sub testDsn {  sub testDsn {
388    my $self = shift;    my $self = shift;
389    my $dsn = $self->{locator}->{dbi}->{dsn};    my $dsn = $self->{locator}->{dbi}->{dsn};
# Line 257  sub testDsn { Line 391  sub testDsn {
391    if ( my $dbh = DBI->connect($dsn, '', '', {    if ( my $dbh = DBI->connect($dsn, '', '', {
392                                                        PrintError => 0,                                                        PrintError => 0,
393                                                        } ) ) {                                                        } ) ) {
394        
395        # TODO: REVIEW
396      $dbh->disconnect();      $dbh->disconnect();
397        
398      return 1;      return 1;
399    } else {    } else {
400      $logger->error( __PACKAGE__ .  "[$self->{locator}->{type}]" . "->testDsn(): " . "DBI-error: " . $DBI::errstr );      $logger->warning( __PACKAGE__ .  "[$self->{locator}->{type}]" . "->testDsn(): " . "DBI-error: " . $DBI::errstr );
401    }    }
402  }  }
403    
 sub createDb {  
   my $self = shift;  
   my $dsn = $self->{locator}->{dbi}->{dsn};  
   
   $logger->debug( __PACKAGE__ .  "->createDb( dsn $dsn )" );  
   
   $dsn =~ s/database=(.+?);//;  
   my $database_name = $1;  
   
   my $ok;  
     
   if ( my $dbh = DBI->connect($dsn, '', '', {  
                                                       PrintError => 0,  
                                                       } ) ) {  
     if ($database_name) {  
       if ($dbh->do("CREATE DATABASE $database_name;")) {  
         $ok = 1;  
       }  
     }  
     $dbh->disconnect();  
   }  
     
   return $ok;  
     
 }  
   
404  sub dropDb {  sub dropDb {
405    my $self = shift;    my $self = shift;
406    my $dsn = $self->{locator}->{dbi}->{dsn};    my $dsn = $self->{locator}->{dbi}->{dsn};
# Line 309  sub dropDb { Line 420  sub dropDb {
420          $ok = 1;          $ok = 1;
421        }        }
422      }      }
423    
424      $dbh->disconnect();      $dbh->disconnect();
425    
426    }    }
427        
428    return $ok;    return $ok;
429  }  }
430    
 sub isConnected {  
   my $self = shift;  
   return 1 if $self->{STORAGEHANDLE};  
 }  
   
431  1;  1;
432  __END__  __END__
433    
434    
435  =head1 DESCRIPTION  =head1 DESCRIPTION
436    
437  Data::Storage is module for a accessing various "data structures" stored inside  =head2 Data::Storage
 various "data containers". It sits on top of DBI and/or Tangram.  
438    
439      Data::Storage is a module for accessing various "data structures / kinds of structured data" stored inside
440      various "data containers".
441      We tried to use the AdapterPattern (http://c2.com/cgi/wiki?AdapterPattern) to implement a wrapper-layer
442      around core CPAN modules (Tangram, DBI).
443    
444    =head2 Why?
445    
446      You will get a better code-structure (not bad for later maintenance) in growing Perl code projects,
447      especially when using multiple database connections at the same time.
448      You will be able to switch between different _kinds_ of implementations used for storing data.
449      Your code will use the very same API to access these storage layers.
450          ... implementation has to be changed for now
451      Maybe you will be able to switch "on-the-fly" without changing any bits in code in the future....
452          ... but that's not the focus
453    
454  =head1 AUTHORS / COPYRIGHT  =head2 What else?
455    
456  The Data::Storage module is Copyright (c) 2002 Andreas Motl.    Having this, we were able to do implement a generic data synchronization module more easy,
457  All rights reserved.    please look at Data::Transfer.
458    
459  You may distribute it under the terms of either the GNU General Public  
460  License or the Artistic License, as specified in the Perl README file.  =head1 AUTHORS / COPYRIGHT
461    
462      The Data::Storage module is Copyright (c) 2002 Andreas Motl.
463      All rights reserved.
464      You may distribute it under the terms of either the GNU General Public
465      License or the Artistic License, as specified in the Perl README file.
466    
467    
468  =head1 ACKNOWLEDGEMENTS  =head1 ACKNOWLEDGEMENTS
469    
470  Larry Wall and the C<perl5-porters> for Perl,    Larry Wall for Perl, Tim Bunce for DBI, Jean-Louis Leroy for Tangram and Set::Object,
471  Tim Bunce for DBI, Jean-Louis Leroy for Tangram and Set::Object,    Sam Vilain for Class::Tangram, Jochen Wiedmann and Jeff Zucker for DBD::CSV & Co.,
472  Sam Vilain for Class::Tangram.    Adam Spiers for MySQL::Diff and all contributors.
473    
474    
475  =head1 SUPPORT / WARRANTY  =head1 SUPPORT / WARRANTY
476    
477  Data::Storage is free software. IT COMES WITHOUT WARRANTY OF ANY KIND.    Data::Storage is free software. IT COMES WITHOUT WARRANTY OF ANY KIND.
478    
479    
480  =head1 TODO  =head1 TODO
481    
482    
483  =head2 Handle the following errors/cases:  =head2 BUGS
484    
485  =head3 "DBI-Error [Tangram]: DBD::mysql::st execute failed: Unknown column 't1.requestdump' in 'field list'"  "DBI-Error [Tangram]: DBD::mysql::st execute failed: Unknown column 't1.requestdump' in 'field list'"
486    
487      ... occours when operating on object-attributes not introduced yet:    ... occours when operating on object-attributes not introduced yet:
488      this should be detected and appended/replaced through:    this should be detected and appended/replaced through:
489      "Schema-Error detected, maybe (just) an inconsistency.    "Schema-Error detected, maybe (just) an inconsistency.
490      Please check if your declaration in schema-module "a" matches structure in database "b" or try to run"    Please check if your declaration in schema-module "a" matches structure in database "b" or try to run"
491      db_setup.pl --dbkey=import --action=deploy    db_setup.pl --dbkey=import --action=deploy
492    
493  =head3 Compare schema (structure diff) with database ...  
494    Compare schema (structure diff) with database ...
495    
496    ... when issuing "db_setup.pl --dbkey=import --action=deploy"    ... when issuing "db_setup.pl --dbkey=import --action=deploy"
497    on a database with an already deployed schema, use an additional "--update" then    on a database with an already deployed schema, use an additional "--update" then
# Line 393  Data::Storage is free software. IT COMES Line 520  Data::Storage is free software. IT COMES
520    automatically and this is believed to be the most common case under normal circumstances.    automatically and this is believed to be the most common case under normal circumstances.
521    
522    
523  =head2 Introduce some features:  =head2 FEATURES
524    
525    - Get this stuff together with UML (Unified Modeling Language) and/or standards from ODMG.    - Get this stuff together with UML (Unified Modeling Language) and/or standards from ODMG.
526    - Make it possible to load/save schemas in XMI (XML Metadata Interchange),    - Make it possible to load/save schemas in XMI (XML Metadata Interchange),
# Line 401  Data::Storage is free software. IT COMES Line 528  Data::Storage is free software. IT COMES
528      Integrate/bundle this with a web-/html-based UML modeling tool or      Integrate/bundle this with a web-/html-based UML modeling tool or
529      some other interesting stuff like the "Co-operative UML Editor" from Uni Darmstadt. (web-/java-based)      some other interesting stuff like the "Co-operative UML Editor" from Uni Darmstadt. (web-/java-based)
530    - Enable Round Trip Engineering. Keep code and diagrams in sync. Don't annoy/bother the programmers.    - Enable Round Trip Engineering. Keep code and diagrams in sync. Don't annoy/bother the programmers.
531    - Add some more handlers:    - Add support for some more handlers/locators to be able to
532      - look at DBD::CSV, Text::CSV, XML::CSV, XML::Excel       access the following standards/protocols/interfaces/programs/apis transparently:
533    - Add some more locations/locators:      +  DBD::CSV (via Data::Storage::Handler::DBI)
534      - PerlDAV: http://www.webdav.org/perldav/     (-) Text::CSV, XML::CSV, XML::Excel
535    - Move to t3, use InCASE      -  MAPI
536        -  LDAP
537        -  DAV (look at PerlDAV: http://www.webdav.org/perldav/)
538        -  Mbox (use formail for seperating/splitting entries/nodes)
539        -  Cyrus (cyrdeliver - what about cyrretrieve (export)???)
540        -  use File::DiffTree, use File::Compare
541        -  Hibernate
542        -  "Win32::UserAccountDb"
543        -  "*nix::UserAccountDb"
544        -  .wab - files (Windows Address Book)
545        -  .pst - files (Outlook Post Storage?)
546        -  XML (e.g. via XML::Simple?)
547      - Move to t3, look at InCASE
548      - some kind of security layer for methods/objects
549        - acls (stored via tangram/ldap?) for functions, methods and objects (entity- & data!?)
550        - where are the hooks needed then?
551          - is Data::Storage & Co. okay, or do we have to touch the innards of DBI and/or Tangram?
552          - an attempt to start could be:
553             - 'sub getACLByObjectId($id, $context)'
554             - 'sub getACLByMethodname($id, $context)'
555             - 'sub getACLByName($id, $context)'
556                ( would require a kinda registry to look up these very names pointing to arbitrary locations (code, data, ...) )
557    
558    
559    
560  =head3 Links:  =head3 LINKS / REFERENCES
561    
562    Specs:    Specs:
563      UML 1.3 Spec: http://cgi.omg.org/cgi-bin/doc?ad/99-06-08.pdf      UML 1.3 Spec: http://cgi.omg.org/cgi-bin/doc?ad/99-06-08.pdf
# Line 434  Data::Storage is free software. IT COMES Line 583  Data::Storage is free software. IT COMES
583      (Dia (free): http://www.lysator.liu.se/~alla/dia/)      (Dia (free): http://www.lysator.liu.se/~alla/dia/)
584      UMLet (free, university): http://www.swt.tuwien.ac.at/umlet/index.html      UMLet (free, university): http://www.swt.tuwien.ac.at/umlet/index.html
585      Voodoo (free): http://voodoo.sourceforge.net/      Voodoo (free): http://voodoo.sourceforge.net/
586        Umbrello UML Modeller: http://uml.sourceforge.net/
587    
588    UML Tools:    UML Tools:
589      http://www.objectsbydesign.com/tools/umltools_byPrice.html      http://www.objectsbydesign.com/tools/umltools_byPrice.html
Legend:
Removed from v.1.5  
changed lines
  Added in v.1.12
MailToCvsAdmin">MailToCvsAdmin
 ViewVC Help
Powered by ViewVC 1.1.26 RSS 2.0 feed