/[cvs]/nfo/perl/libs/Data/Transfer/Sync.pm
ViewVC logotype

Diff of /nfo/perl/libs/Data/Transfer/Sync.pm

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 1.3 by joko, Sun Dec 1 22:26:59 2002 UTC revision 1.7 by joko, Fri Dec 13 21:49:34 2002 UTC
# Line 6  Line 6 
6  ##  ##
7  ##    ----------------------------------------------------------------------------------------  ##    ----------------------------------------------------------------------------------------
8  ##    $Log$  ##    $Log$
9    ##    Revision 1.7  2002/12/13 21:49:34  joko
10    ##    + sub configure
11    ##    + sub checkOptions
12    ##
13    ##    Revision 1.6  2002/12/06 04:49:10  jonen
14    ##    + disabled output-puffer here
15    ##
16    ##    Revision 1.5  2002/12/05 08:06:05  joko
17    ##    + bugfix with determining empty fields (Null) with DBD::CSV
18    ##    + debugging
19    ##    + updated comments
20    ##
21    ##    Revision 1.4  2002/12/03 15:54:07  joko
22    ##    + {import}-flag is now {prepare}-flag
23    ##
24  ##    Revision 1.3  2002/12/01 22:26:59  joko  ##    Revision 1.3  2002/12/01 22:26:59  joko
25  ##    + minor cosmetics for logging  ##    + minor cosmetics for logging
26  ##  ##
# Line 31  use strict; Line 46  use strict;
46  use warnings;  use warnings;
47    
48  use Data::Dumper;  use Data::Dumper;
49    #use Hash::Merge qw( merge );
50    
51  use misc::HashExt;  use misc::HashExt;
52  use libp qw( md5_base64 );  use libp qw( md5_base64 );
53  use libdb qw( quotesql hash2Sql );  use libdb qw( quotesql hash2Sql );
# Line 40  use Data::Compare::Struct qw( getDiffere Line 57  use Data::Compare::Struct qw( getDiffere
57  # get logger instance  # get logger instance
58  my $logger = Log::Dispatch::Config->instance;  my $logger = Log::Dispatch::Config->instance;
59    
60    $| = 1;
61    
62  sub new {  sub new {
63    my $invocant = shift;    my $invocant = shift;
64    my $class = ref($invocant) || $invocant;    my $class = ref($invocant) || $invocant;
65    my $self = { @_ };    my $self = {};
66    $logger->debug( __PACKAGE__ . "->new(@_)" );    $logger->debug( __PACKAGE__ . "->new(@_)" );
67    bless $self, $class;    bless $self, $class;
68    $self->_init();    $self->configure(@_);
69    return $self;    return $self;
70  }  }
71    
72    
73    sub configure {
74      my $self = shift;
75      my @args = @_;
76      if (!isEmpty(\@args)) {
77        my %properties = @_;
78        # merge args to properties
79        map { $self->{$_} = $properties{$_}; } keys %properties;
80        $self->_init();
81      } else {
82        #print "no args!", "\n";
83      }
84      #print Dumper($self);
85    }
86    
87  sub _init {  sub _init {
88    my $self = shift;    my $self = shift;
89    
90      $self->{configured} = 1;
91        
92    # build new container if necessary    # build new container if necessary
93    $self->{container} = Data::Storage::Container->new() if !$self->{container};    $self->{container} = Data::Storage::Container->new() if !$self->{container};
# Line 72  sub _init { Line 106  sub _init {
106  }  }
107    
108    
109    sub prepareOptions {
110    
111      my $self = shift;
112      my $opts = shift;
113    
114    #print Dumper($opts);
115    
116      $opts->{mode} ||= '';
117      $opts->{erase} ||= 0;
118      #$opts->{import} ||= 0;
119      
120      $logger->info( __PACKAGE__ . "->prepareOptions( source_node $opts->{source_node} mode $opts->{mode} erase $opts->{erase} prepare $opts->{prepare} )");
121    
122      if (!$opts->{mapping} || !$opts->{mapping_module}) {
123        $logger->warning( __PACKAGE__ . "->prepareOptions: No mapping supplied - please check key 'mappings' in BizWorks/Config.pm");
124      }
125    
126      my $evstring = "use $opts->{mapping_module};";
127      eval($evstring);
128      if ($@) {
129        $logger->warning( __PACKAGE__ . "->prepareOptions: error while trying to access mapping - $@");
130        return;
131      }
132    
133      # resolve mapping metadata (returned from sub)
134      my $mapObject = $opts->{mapping_module}->new();
135      #print Dumper($map);
136      my $source_node_name = $opts->{source_node};
137      # check if mapping for certain node is contained in mapping object
138      if (!$mapObject->can($source_node_name)) {
139        $logger->warning( __PACKAGE__ . "->prepareOptions: Can't access mapping for node \"$source_node_name\" - please check $opts->{mapping_module}.");
140        return;
141      }
142      my $map = $mapObject->$source_node_name;
143    
144      # remove asymmetries from $map (patch keys)
145      $map->{source_node} = $map->{source}; delete $map->{source};
146      $map->{target_node} = $map->{target}; delete $map->{target};
147      $map->{mapping} = $map->{details}; delete $map->{details};
148      $map->{direction} = $map->{mode}; delete $map->{mode};
149    
150      # defaults (mostly for backward-compatibility)
151      $map->{source_node} ||= $source_node_name;
152      $map->{source_ident} ||= 'storage_method:id';
153      $map->{target_ident} ||= 'property:oid';
154      $map->{direction} ||= $opts->{mode};         # | PUSH | PULL | FULL
155      $map->{method} ||= 'checksum';                # | timestamp
156      $map->{source_exclude} ||= [qw( cs )];
157    
158      # merge map to opts
159      map { $opts->{$_} = $map->{$_}; } keys %$map;
160        
161    #print Dumper($opts);
162    
163      # TODO: move this to checkOptions...
164      
165      # check - do we have a target?
166      if (!$opts->{target_node}) {
167        $logger->warning( __PACKAGE__ . "->prepareOptions: No target given - please check metadata declaration.");
168        return;
169      }
170    
171    
172      #return $opts;
173      return 1;
174    
175    }
176    
177    
178    sub checkOptions {
179      my $self = shift;
180      my $opts = shift;
181      
182      my $result = 1;
183      
184      # check - do we have a target node?
185      if (!$opts->{target_node}) {
186        $logger->warning( __PACKAGE__ . "->checkOptions: Error while resolving resource metadata - no 'target node' could be determined.");
187        $result = 0;
188      }
189    
190      # check - do we have a mapping?
191      if (!$opts->{mapping} && !$opts->{mapping_module}) {
192        $logger->warning( __PACKAGE__ . "->checkOptions: Error while resolving resource metadata - no 'mapping' could be determined.");
193        $result = 0;
194      }
195      
196      return $result;
197      
198    }
199    
200    
201  # TODO: some feature to show off the progress of synchronization (cur/max * 100)  # TODO: some feature to show off the progress of synchronization (cur/max * 100)
202  sub syncNodes {  sub syncNodes {
203    
204    my $self = shift;    my $self = shift;
205    my $args = shift;    my $args = shift;
206    
207      if (!$self->{configured}) {
208        $logger->critical( __PACKAGE__ . "->syncNodes: Synchronization object is not configured/initialized correctly." );
209        return;
210      }
211    
212    # remember arguments through the whole processing    # remember arguments through the whole processing
213    $self->{args} = $args;    $self->{args} = $args;
214    
# Line 100  sub syncNodes { Line 231  sub syncNodes {
231    }    }
232    
233    # decompose identifiers for each partner    # decompose identifiers for each partner
234    # TODO: take this list from already established/given metadata    # TODO: refactor!!! take this list from already established/given metadata
235    foreach ('source', 'target') {    foreach ('source', 'target') {
236            
237      # get/set metadata for further processing      # get/set metadata for further processing
# Line 155  sub syncNodes { Line 286  sub syncNodes {
286      #print "iiiiisprov: ", Dumper($self->{meta}->{$_}->{storage}), "\n";      #print "iiiiisprov: ", Dumper($self->{meta}->{$_}->{storage}), "\n";
287    }    }
288    
289    #print Dumper($self->{meta});
290    
291    $logger->info( __PACKAGE__ . "->syncNodes: source=$self->{meta}->{source}->{dbkey}/$self->{meta}->{source}->{node} $direction_arrow target=$self->{meta}->{target}->{dbkey}/$self->{meta}->{target}->{node}" );    $logger->info( __PACKAGE__ . "->syncNodes: source=$self->{meta}->{source}->{dbkey}/$self->{meta}->{source}->{node} $direction_arrow target=$self->{meta}->{target}->{dbkey}/$self->{meta}->{target}->{node}" );
292    
293    # build mapping    # build mapping
# Line 182  sub syncNodes { Line 315  sub syncNodes {
315    
316    }    }
317    
318    #print Dumper($self->{meta});
319      
320    # check partners/nodes: does partner exist / is node available?    # check partners/nodes: does partner exist / is node available?
321    foreach my $partner (keys %{$self->{meta}}) {    foreach my $partner (keys %{$self->{meta}}) {
322      next if $self->{meta}->{$partner}->{storage}->{locator}->{type} eq 'DBI';    # for DBD::CSV - re-enable for others      next if $self->{meta}->{$partner}->{storage}->{locator}->{type} eq 'DBI';    # for DBD::CSV - re-enable for others
# Line 213  sub syncNodes { Line 348  sub syncNodes {
348    
349    # import flag means: prepare the source node to be syncable    # import flag means: prepare the source node to be syncable
350    # this is useful if there are e.g. no "ident" or "checksum" columns yet inside a DBI like (row-based) storage    # this is useful if there are e.g. no "ident" or "checksum" columns yet inside a DBI like (row-based) storage
351    if ($self->{args}->{import}) {    if ($self->{args}->{prepare}) {
352      $self->_prepareNode_MetaProperties('source');      $self->_prepareNode_MetaProperties('source');
353      $self->_prepareNode_DummyIdent('source');      $self->_prepareNode_DummyIdent('source');
354      #return;      #return;
# Line 252  sub _syncNodes { Line 387  sub _syncNodes {
387      $results ||= $self->_getNodeList('source', $filter);      $results ||= $self->_getNodeList('source', $filter);
388    }    }
389        
390    # get reference to node list from convenient method provided by corehandle    # get reference to node list from convenient method provided by CORE-HANDLE
391    #$results ||= $self->{source}->getListUnfiltered($self->{meta}->{source}->{node});    #$results ||= $self->{source}->getListUnfiltered($self->{meta}->{source}->{node});
392    #$results ||= $self->{meta}->{source}->{storage}->getListUnfiltered($self->{meta}->{source}->{node});    #$results ||= $self->{meta}->{source}->{storage}->getListUnfiltered($self->{meta}->{source}->{node});
393    $results ||= $self->_getNodeList('source');    $results ||= $self->_getNodeList('source');
# Line 330  sub _syncNodes { Line 465  sub _syncNodes {
465      my $identOK = $self->_resolveNodeIdent('source');      my $identOK = $self->_resolveNodeIdent('source');
466      #if (!$identOK && lc $self->{args}->{direction} ne 'import') {      #if (!$identOK && lc $self->{args}->{direction} ne 'import') {
467      if (!$identOK) {      if (!$identOK) {
468        $logger->critical( __PACKAGE__ . "->syncNodes: Can not synchronize: No ident found in source node, maybe try to \"import\" this node first." );        #print Dumper($self->{meta}->{source});
469          $logger->critical( __PACKAGE__ . "->syncNodes: No ident found in source node \"$self->{meta}->{source}->{node}\", try to \"prepare\" this node first?" );
470        return;        return;
471      }      }
472    
473  #print "statload", "\n";  #print "statload", "\n";
474  #print "ident: ", $self->{node}->{source}->{ident}, "\n";  #print "ident: ", $self->{node}->{source}->{ident}, "\n";
475    #print Dumper($self->{node});
476            
477      my $statOK = $self->_statloadNode('target', $self->{node}->{source}->{ident});      my $statOK = $self->_statloadNode('target', $self->{node}->{source}->{ident});
478    
479    #print Dumper($self->{node});
480            
481      # mark node as new either if there's no ident or if stat/load failed      # mark node as new either if there's no ident or if stat/load failed
482      if (!$statOK) {      if (!$statOK) {
# Line 424  sub _syncNodes { Line 563  sub _syncNodes {
563        $tc->{attempt_new}++;        $tc->{attempt_new}++;
564        $self->_doTransferToTarget('insert');        $self->_doTransferToTarget('insert');
565        # asymmetry: refetch node from target to re-calculate new ident and checksum (TODO: is IdentAuthority of relevance here?)        # asymmetry: refetch node from target to re-calculate new ident and checksum (TODO: is IdentAuthority of relevance here?)
566          #print Dumper($self->{node});
567        $self->_statloadNode('target', $self->{node}->{target}->{ident}, 1);        $self->_statloadNode('target', $self->{node}->{target}->{ident}, 1);
568        $self->_readChecksum('target');        $self->_readChecksum('target');
569    
# Line 449  sub _syncNodes { Line 589  sub _syncNodes {
589      # change ident in source (take from target), if transfer was ok and target is an IdentAuthority      # change ident in source (take from target), if transfer was ok and target is an IdentAuthority
590      # this is (for now) called a "retransmit" indicated by a "r"-character when verbosing      # this is (for now) called a "retransmit" indicated by a "r"-character when verbosing
591      if ($self->{node}->{status}->{ok} && $self->{meta}->{target}->{storage}->{isIdentAuthority}) {      if ($self->{node}->{status}->{ok} && $self->{meta}->{target}->{storage}->{isIdentAuthority}) {
592          print "r" if $self->{verbose};
593        #print Dumper($self->{meta});        #print Dumper($self->{meta});
594        #print Dumper($self->{node});        #print Dumper($self->{node});
595        #exit;        #exit;
596        $self->_doModifySource_IdentChecksum($self->{node}->{target}->{ident});        $self->_doModifySource_IdentChecksum($self->{node}->{target}->{ident});
       print "r" if $self->{verbose};  
597      }      }
598    
599      print ":" if $self->{verbose};      print ":" if $self->{verbose};
# Line 491  sub _dumpCompact { Line 631  sub _dumpCompact {
631      my $item = {};      my $item = {};
632      foreach my $key (keys %$_) {      foreach my $key (keys %$_) {
633        my $val = $_->{$key};        my $val = $_->{$key};
634    
635    #print Dumper($val);
636    
637        if (ref $val eq 'Set::Object') {        if (ref $val eq 'Set::Object') {
638          #print "========================= SET", "\n";          #print "========================= SET", "\n";
639          #print Dumper($val);  #print Dumper($val);
640          #print Dumper($val->members());          #print Dumper($val->members());
641          #$val = $val->members();          #$val = $val->members();
642          #$vars->[$count]->{$key} = $val->members() if $val->can("members");          #$vars->[$count]->{$key} = $val->members() if $val->can("members");
643          #$item->{$key} = $val->members() if $val->can("members");          #$item->{$key} = $val->members() if $val->can("members");
644          $item->{$key} = $val->members();          $item->{$key} = $val->members();
645          #print Dumper($vars->[$count]->{$key});          #print Dumper($vars->[$count]->{$key});
646    
647        } else {        } else {
648          $item->{$key} = $val;          $item->{$key} = $val;
649        }        }
650    
651      }      }
652      push @data, $item;      push @data, $item;
653      $count++;      $count++;
654    }    }
655    
656  #print "Dump:", "\n";  #print "Dump:", Dumper(@data), "\n";
 #print Dumper(@data);  
657    
658    $Data::Dumper::Indent = 0;    $Data::Dumper::Indent = 0;
659    my $result = Dumper(@data);    my $result = Dumper(@data);
660    $Data::Dumper::Indent = 2;    $Data::Dumper::Indent = 2;
661    return $result;    return $result;
662      
663  }  }
664    
665    
# Line 744  sub _modifyNode { Line 889  sub _modifyNode {
889      }      }
890    }    }
891        
892      
893      #print Dumper($self->{meta});
894    
895    # DBI speaks SQL    # DBI speaks SQL
896    if ($self->{meta}->{$descent}->{storage}->{locator}->{type} eq 'DBI') {    if ($self->{meta}->{$descent}->{storage}->{locator}->{type} eq 'DBI') {
# Line 757  sub _modifyNode { Line 904  sub _modifyNode {
904      #print $action, "\n";      #print $action, "\n";
905  #$action = "anc";  #$action = "anc";
906  #print "yai", "\n";  #print "yai", "\n";
907    
908    #print Dumper($map);
909    #delete $map->{cs};
910    
911      if (lc($action) eq 'insert') {      if (lc($action) eq 'insert') {
912        $sql_main = hash2Sql($self->{meta}->{$descent}->{node}, $map, 'SQL_INSERT');        $sql_main = hash2Sql($self->{meta}->{$descent}->{node}, $map, 'SQL_INSERT');
913      } elsif (lc $action eq 'update') {      } elsif (lc $action eq 'update') {
# Line 764  sub _modifyNode { Line 915  sub _modifyNode {
915        $sql_main = hash2Sql($self->{meta}->{$descent}->{node}, $map, 'SQL_UPDATE', $crit);        $sql_main = hash2Sql($self->{meta}->{$descent}->{node}, $map, 'SQL_UPDATE', $crit);
916      }      }
917    
918      #print "sql: ", $sql_main, "\n";  #$sql_main = "UPDATE currencies_csv SET oid='abcdef' WHERE text='Australian Dollar' AND key='AUD';";
919      #exit;  #$sql_main = "UPDATE currencies_csv SET oid='huhu2' WHERE ekey='AUD'";
920    
921    #print "sql: ", $sql_main, "\n";
922    #exit;
923    
924      # transfer data      # transfer data
925      my $sqlHandle = $self->{meta}->{$descent}->{storage}->sendCommand($sql_main);      my $sqlHandle = $self->{meta}->{$descent}->{storage}->sendCommand($sql_main);
926    
927    #exit;
928    
929      # handle errors      # handle errors
930      if ($sqlHandle->err) {      if ($sqlHandle->err) {
931        #if ($self->{args}->{debug}) { print "sql-error with statement: $sql_main", "\n"; }        #if ($self->{args}->{debug}) { print "sql-error with statement: $sql_main", "\n"; }
# Line 845  sub _modifyNode { Line 1001  sub _modifyNode {
1001        hash2object($object, $map);        hash2object($object, $map);
1002    
1003        # ... and re-update@orm.        # ... and re-update@orm.
1004    #print Dumper($object);
1005        $self->{meta}->{$descent}->{storage}->update($object);        $self->{meta}->{$descent}->{storage}->update($object);
1006    
1007        # asymmetry: get ident after insert        # asymmetry: get ident after insert
1008        # TODO:        # TODO:
1009        #   - just do this if it is an IdentAuthority        #   - just do this if it is an IdentAuthority
1010        #   - use IdentProvider metadata here        #   - use IdentProvider metadata here
1011        $self->{node}->{$descent}->{ident} = $self->{meta}->{$descent}->{storage}->id($object);  #print Dumper($self->{meta}->{$descent});
1012          my $oid = $self->{meta}->{$descent}->{storage}->id($object);
1013    #print "oid: $oid", "\n";
1014          $self->{node}->{$descent}->{ident} = $oid;
1015    
1016    
1017      } elsif (lc $action eq 'update') {      } elsif (lc $action eq 'update') {
# Line 948  sub _statloadNode { Line 1108  sub _statloadNode {
1108        #print "\n", "Attempt to fetch entry implicitely by ident failed: no ident given! This may result in an insert if no write-protection is in the way.", "\n";        #print "\n", "Attempt to fetch entry implicitely by ident failed: no ident given! This may result in an insert if no write-protection is in the way.", "\n";
1109        return;        return;
1110      }      }
1111        
1112        # patch for DBD::CSV
1113        if ($ident && $ident eq 'Null') {
1114          return;
1115        }
1116    
1117      my $result = $self->{meta}->{$descent}->{storage}->sendQuery({  #print "yai!", "\n";
1118    
1119        my $query = {
1120        node => $self->{meta}->{$descent}->{node},        node => $self->{meta}->{$descent}->{node},
1121        subnodes => [qw( cs )],        subnodes => [qw( cs )],
1122        criterias => [        criterias => [
# Line 957  sub _statloadNode { Line 1124  sub _statloadNode {
1124             op => 'eq',             op => 'eq',
1125             val => $ident },             val => $ident },
1126        ]        ]
1127      });      };
1128    
1129    #print Dumper($query);
1130    
1131        my $result = $self->{meta}->{$descent}->{storage}->sendQuery($query);
1132    
1133      my $entry = $result->getNextEntry();      my $entry = $result->getNextEntry();
1134    
1135    #print Dumper($entry);
1136    #print "pers: " . $self->{meta}->{$descent}->{storage}->is_persistent($entry), "\n";
1137    #my $state = $self->{meta}->{$descent}->{storage}->_fetch_object_state($entry, { name => 'TransactionHop' } );
1138    #print Dumper($state);
1139    
1140      my $status = $result->getStatus();      my $status = $result->getStatus();
1141    
1142    #print Dumper($status);
1143        
1144      # TODO: enhance error handling (store inside tc)      # TODO: enhance error handling (store inside tc)
1145      #if (!$row) {      #if (!$row) {
1146      #  print "\n", "row error", "\n";      #  print "\n", "row error", "\n";
1147      #  next;      #  next;
1148      #}      #}
1149      if (($status && $status->{err}) || !$entry) {  
1150        #$logger->critical( __PACKAGE__ . "->_loadNode (ident=\"$ident\") failed" );      # these checks run before actually loading payload- and meta-data to node-container
1151        return;      
1152      }        # 1st level - hard error
1153          if ($status && $status->{err}) {
1154            $logger->debug( __PACKAGE__ . "->_statloadNode (ident=\"$ident\") failed - hard error (that's ok): $status->{err}" );
1155            return;
1156          }
1157      
1158          # 2nd level - logical (empty/notfound) error
1159          if (($status && $status->{empty}) || !$entry) {
1160            $logger->debug( __PACKAGE__ . "->_statloadNode (ident=\"$ident\") failed - logical error (that's ok)" );
1161            #print "no entry (logical)", "\n";
1162            return;
1163          }
1164    
1165    #print Dumper($entry);
1166    
1167      # was:      # was:
1168      # $self->{node}->{$descent}->{ident} = $ident;        # $self->{node}->{$descent}->{ident} = $ident;  
1169      # is:      # is:
1170      # TODO: re-resolve ident from entry via metadata "IdentProvider"      # TODO: re-resolve ident from entry via metadata "IdentProvider" here - like elsewhere
1171      $self->{node}->{$descent}->{ident} = $ident;      $self->{node}->{$descent}->{ident} = $ident;
1172      $self->{node}->{$descent}->{payload} = $entry;      $self->{node}->{$descent}->{payload} = $entry;
1173    
1174    }    }
1175        
1176    return 1;    return 1;
# Line 1000  sub _doModifySource_IdentChecksum { Line 1194  sub _doModifySource_IdentChecksum {
1194      $self->{meta}->{source}->{IdentProvider}->{arg} => $ident_new,      $self->{meta}->{source}->{IdentProvider}->{arg} => $ident_new,
1195      cs => $self->{node}->{target}->{checksum},      cs => $self->{node}->{target}->{checksum},
1196    };    };
1197    #print Dumper($map);  
1198    #print Dumper($self->{node});  #print Dumper($map);
1199    #exit;  #print Dumper($self->{node});
1200    #exit;
1201    
1202    $self->_modifyNode('source', 'update', $map);    $self->_modifyNode('source', 'update', $map);
1203  }  }
1204    
# Line 1084  sub _prepareNode_DummyIdent { Line 1280  sub _prepareNode_DummyIdent {
1280      }      }
1281      my $crit = join ' AND ', @crits;      my $crit = join ' AND ', @crits;
1282      print "p" if $self->{verbose};      print "p" if $self->{verbose};
1283    
1284    #print Dumper($map);
1285    #print Dumper($crit);
1286    
1287      $self->_modifyNode($descent, 'update', $map, $crit);      $self->_modifyNode($descent, 'update', $map, $crit);
1288      $i++;      $i++;
1289    }    }
Legend:
Removed from v.1.3  
changed lines
  Added in v.1.7
MailToCvsAdmin">MailToCvsAdmin
 ViewVC Help
Powered by ViewVC 1.1.26 RSS 2.0 feed