monotone

monotone Mtn Source Tree

Root/schema_migration.cc

1// Copyright (C) 2002 Graydon Hoare <graydon@pobox.com>
2//
3// This program is made available under the GNU GPL version 2.0 or
4// greater. See the accompanying file COPYING for details.
5//
6// This program is distributed WITHOUT ANY WARRANTY; without even the
7// implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
8// PURPOSE.
9
10#include <algorithm>
11#include <string>
12#include <vector>
13#include <locale>
14#include <stdexcept>
15#include <iostream>
16#include <map>
17
18#include <boost/tokenizer.hpp>
19
20#include <sqlite3.h>
21
22#include "sanity.hh"
23#include "schema_migration.hh"
24#include "botan/botan.h"
25#include "app_state.hh"
26#include "keys.hh"
27#include "transforms.hh"
28
29using std::ctype;
30using std::locale;
31using std::map;
32using std::pair;
33using std::remove_if;
34using std::string;
35using std::use_facet;
36using std::vector;
37
38using boost::char_separator;
39
40// this file knows how to migrate schema databases. the general strategy is
41// to hash each schema we ever use, and make a list of the SQL commands
42// required to get from one hash value to the next. when you do a
43// migration, the migrator locates your current db's state on the list and
44// then runs all the migration functions between that point and the target
45// of the migration.
46
47// you will notice a little bit of duplicated code between here and
48// transforms.cc / database.cc; this was originally to facilitate inclusion of
49// migration capability into the depot code, which did not link against those
50// objects. the depot code is gone, but this isn't the sort of code that
51// should ever be touched after being written, so the duplication currently
52// remains. if it becomes a maintainence burden, however, consider
53// refactoring.
54
55static int logged_sqlite3_exec(sqlite3* db,
56 const char* sql,
57 sqlite3_callback cb,
58 void* data,
59 char** errmsg)
60{
61 L(FL("executing SQL '%s'") % sql);
62 int res = sqlite3_exec(db, sql, cb, data, errmsg);
63 L(FL("result: %i (%s)") % res % sqlite3_errmsg(db));
64 if (errmsg && ((*errmsg)!=0))
65 L(FL("errmsg: %s") % *errmsg);
66 return res;
67}
68
69typedef boost::tokenizer<char_separator<char> > tokenizer;
70
71static string
72lowercase_facet(string const & in)
73{
74 I(40==in.size());
75 const int sz=40;
76 char buf[sz];
77 in.copy(buf, sz);
78 locale loc;
79 use_facet< ctype<char> >(loc).tolower(buf, buf+sz);
80 return string(buf,sz);
81}
82
83static void
84massage_sql_tokens(string const & in,
85 string & out)
86{
87 char_separator<char> sep(" \r\n\t", "(),;");
88 tokenizer tokens(in, sep);
89 out.clear();
90 for (tokenizer::iterator i = tokens.begin();
91 i != tokens.end(); ++i)
92 {
93 if (i != tokens.begin())
94 out += " ";
95 out += *i;
96 }
97}
98
99static void
100calculate_id(string const & in,
101 string & ident)
102{
103 Botan::Pipe p(new Botan::Hash_Filter("SHA-1"), new Botan::Hex_Encoder());
104 p.process_msg(in);
105
106 ident = lowercase_facet(p.read_all_as_string());
107}
108
109
110struct
111is_ws
112{
113 bool operator()(char c) const
114 {
115 return c == '\r' || c == '\n' || c == '\t' || c == ' ';
116 }
117};
118
119static void
120sqlite_sha1_fn(sqlite3_context *f, int nargs, sqlite3_value ** args)
121{
122 string tmp, sha;
123 if (nargs <= 1)
124 {
125 sqlite3_result_error(f, "need at least 1 arg to sha1()", -1);
126 return;
127 }
128
129 if (nargs == 1)
130 {
131 string s = reinterpret_cast<char const*>(sqlite3_value_text(args[0]));
132 s.erase(remove_if(s.begin(), s.end(), is_ws()),s.end());
133 tmp = s;
134 }
135 else
136 {
137 string sep = string(reinterpret_cast<char const*>(sqlite3_value_text(args[0])));
138 string s = reinterpret_cast<char const*>(sqlite3_value_text(args[1]));
139 s.erase(remove_if(s.begin(), s.end(), is_ws()),s.end());
140 tmp = s;
141 for (int i = 2; i < nargs; ++i)
142 {
143 s = string(reinterpret_cast<char const*>(sqlite3_value_text(args[i])));
144 s.erase(remove_if(s.begin(), s.end(), is_ws()),s.end());
145 tmp += sep + s;
146 }
147 }
148 calculate_id(tmp, sha);
149 sqlite3_result_text(f,sha.c_str(),sha.size(),SQLITE_TRANSIENT);
150}
151
152int
153append_sql_stmt(void * vp,
154 int ncols,
155 char ** values,
156 char ** colnames)
157{
158 if (ncols != 1)
159 return 1;
160
161 if (vp == NULL)
162 return 1;
163
164 if (values == NULL)
165 return 1;
166
167 if (values[0] == NULL)
168 return 1;
169
170 string *str = reinterpret_cast<string *>(vp);
171 str->append(values[0]);
172 str->append("\n");
173 return 0;
174}
175
176void
177calculate_schema_id(sqlite3 *sql, string & id)
178{
179 id.clear();
180 string tmp, tmp2;
181 int res = logged_sqlite3_exec(sql,
182 "SELECT sql FROM sqlite_master "
183 "WHERE (type = 'table' OR type = 'index') "
184 // filter out NULL sql statements, because
185 // those are auto-generated indices (for
186 // UNIQUE constraints, etc.).
187 "AND sql IS NOT NULL "
188 "AND name not like 'sqlite_stat%' "
189 "ORDER BY name",
190 &append_sql_stmt, &tmp, NULL);
191 if (res != SQLITE_OK)
192 {
193 // note: useful error messages should be kept consistent with
194 // assert_sqlite3_ok() in database.cc
195 string errmsg(sqlite3_errmsg(sql));
196 L(FL("calculate_schema_id sqlite error: %d: %s") % res % errmsg);
197 string auxiliary_message = "";
198 if (res == SQLITE_ERROR)
199 {
200 auxiliary_message += _("make sure database and containing directory are writeable\n"
201 "and you have not run out of disk space");
202
203 }
204 logged_sqlite3_exec(sql, "ROLLBACK", NULL, NULL, NULL);
205 E(false, F("sqlite error: %s\n%s") % errmsg % auxiliary_message);
206 }
207 massage_sql_tokens(tmp, tmp2);
208 calculate_id(tmp2, id);
209}
210
211typedef bool (*migrator_cb)(sqlite3 *, char **, app_state *);
212
213struct
214migrator
215{
216 vector< pair<string,migrator_cb> > migration_events;
217 app_state * __app;
218
219 void set_app(app_state *app)
220 {
221 __app = app;
222 }
223
224 void add(string schema_id, migrator_cb cb)
225 {
226 migration_events.push_back(make_pair(schema_id, cb));
227 }
228
229 void migrate(sqlite3 *sql, string target_id)
230 {
231 string init;
232
233 I(sql != NULL);
234
235 calculate_schema_id(sql, init);
236
237 I(!sqlite3_create_function(sql, "sha1", -1, SQLITE_UTF8, NULL,
238 &sqlite_sha1_fn, NULL, NULL));
239
240
241 P(F("calculating necessary migration steps"));
242
243 bool migrating = false;
244 for (vector< pair<string, migrator_cb> >::const_iterator i = migration_events.begin();
245 i != migration_events.end(); ++i)
246 {
247
248 if (i->first == init)
249 {
250 E(logged_sqlite3_exec(sql, "BEGIN EXCLUSIVE", NULL, NULL, NULL) == SQLITE_OK,
251 F("error at transaction BEGIN statement"));
252 P(F("migrating data"));
253 migrating = true;
254 }
255
256 if (migrating)
257 {
258 // confirm that we are where we ought to be
259 string curr;
260 char *errmsg = NULL;
261 calculate_schema_id(sql, curr);
262 if (curr != i->first)
263 {
264 logged_sqlite3_exec(sql, "ROLLBACK", NULL, NULL, NULL);
265 I(false);
266 }
267
268 if (i->second == NULL)
269 {
270 logged_sqlite3_exec(sql, "ROLLBACK", NULL, NULL, NULL);
271 I(false);
272 }
273
274 // do this migration step
275 else if (! i->second(sql, &errmsg, __app))
276 {
277 logged_sqlite3_exec(sql, "ROLLBACK", NULL, NULL, NULL);
278 E(false, F("migration step failed: %s")
279 % (errmsg ? errmsg : "unknown error"));
280 }
281 }
282 }
283
284 // confirm that our target schema was met
285 if (migrating)
286 {
287 string curr;
288 calculate_schema_id(sql, curr);
289 if (curr != target_id)
290 {
291 logged_sqlite3_exec(sql, "ROLLBACK", NULL, NULL, NULL);
292 E(false, F("mismatched result of migration, got %s, wanted %s")
293 % curr % target_id);
294 }
295 P(F("committing changes to database"));
296 E(logged_sqlite3_exec(sql, "COMMIT", NULL, NULL, NULL) == SQLITE_OK,
297 F("failure on COMMIT"));
298
299 P(F("optimizing database"));
300 E(logged_sqlite3_exec(sql, "VACUUM", NULL, NULL, NULL) == SQLITE_OK,
301 F("error vacuuming after migration"));
302
303 E(logged_sqlite3_exec(sql, "ANALYZE", NULL, NULL, NULL) == SQLITE_OK,
304 F("error running analyze after migration"));
305 }
306 else
307 {
308 // if we didn't do anything, make sure that it's because we were
309 // already up to date.
310 E(init == target_id,
311 F("database schema %s is unknown; cannot perform migration") % init);
312 // We really want 'db migrate' on an up-to-date schema to be a no-op
313 // (no vacuum or anything, even), so that automated scripts can fire
314 // one off optimistically and not have to worry about getting their
315 // administrators to do it by hand.
316 P(F("no migration performed; database schema already up-to-date at %s") % init);
317 }
318 }
319};
320
321static bool move_table(sqlite3 *sql, char **errmsg,
322 char const * srcname,
323 char const * dstname,
324 char const * dstschema)
325{
326 string create = "CREATE TABLE ";
327 create += dstname;
328 create += " ";
329 create += dstschema;
330
331 int res = logged_sqlite3_exec(sql, create.c_str(), NULL, NULL, errmsg);
332 if (res != SQLITE_OK)
333 return false;
334
335 string insert = "INSERT INTO ";
336 insert += dstname;
337 insert += " SELECT * FROM ";
338 insert += srcname;
339
340 res = logged_sqlite3_exec(sql, insert.c_str(), NULL, NULL, errmsg);
341 if (res != SQLITE_OK)
342 return false;
343
344 string drop = "DROP TABLE ";
345 drop += srcname;
346
347 res = logged_sqlite3_exec(sql, drop.c_str(), NULL, NULL, errmsg);
348 if (res != SQLITE_OK)
349 return false;
350
351 return true;
352}
353
354
355static bool
356migrate_client_merge_url_and_group(sqlite3 * sql,
357 char ** errmsg,
358 app_state *)
359{
360
361 // migrate the posting_queue table
362 if (!move_table(sql, errmsg,
363 "posting_queue",
364 "tmp",
365 "("
366 "url not null,"
367 "groupname not null,"
368 "content not null"
369 ")"))
370 return false;
371
372 int res = logged_sqlite3_exec(sql, "CREATE TABLE posting_queue "
373 "("
374 "url not null, -- URL we are going to send this to\n"
375 "content not null -- the packets we're going to send\n"
376 ")", NULL, NULL, errmsg);
377 if (res != SQLITE_OK)
378 return false;
379
380 res = logged_sqlite3_exec(sql, "INSERT INTO posting_queue "
381 "SELECT "
382 "(url || '/' || groupname), "
383 "content "
384 "FROM tmp", NULL, NULL, errmsg);
385 if (res != SQLITE_OK)
386 return false;
387
388 res = logged_sqlite3_exec(sql, "DROP TABLE tmp", NULL, NULL, errmsg);
389 if (res != SQLITE_OK)
390 return false;
391
392
393 // migrate the incoming_queue table
394 if (!move_table(sql, errmsg,
395 "incoming_queue",
396 "tmp",
397 "("
398 "url not null,"
399 "groupname not null,"
400 "content not null"
401 ")"))
402 return false;
403
404 res = logged_sqlite3_exec(sql, "CREATE TABLE incoming_queue "
405 "("
406 "url not null, -- URL we got this bundle from\n"
407 "content not null -- the packets we're going to read\n"
408 ")", NULL, NULL, errmsg);
409 if (res != SQLITE_OK)
410 return false;
411
412 res = logged_sqlite3_exec(sql, "INSERT INTO incoming_queue "
413 "SELECT "
414 "(url || '/' || groupname), "
415 "content "
416 "FROM tmp", NULL, NULL, errmsg);
417 if (res != SQLITE_OK)
418 return false;
419
420 res = logged_sqlite3_exec(sql, "DROP TABLE tmp", NULL, NULL, errmsg);
421 if (res != SQLITE_OK)
422 return false;
423
424
425 // migrate the sequence_numbers table
426 if (!move_table(sql, errmsg,
427 "sequence_numbers",
428 "tmp",
429 "("
430 "url not null,"
431 "groupname not null,"
432 "major not null,"
433 "minor not null,"
434 "unique(url, groupname)"
435 ")"
436 ))
437 return false;
438
439 res = logged_sqlite3_exec(sql, "CREATE TABLE sequence_numbers "
440 "("
441 "url primary key, -- URL to read from\n"
442 "major not null, -- 0 in news servers, may be higher in depots\n"
443 "minor not null -- last article / packet sequence number we got\n"
444 ")", NULL, NULL, errmsg);
445 if (res != SQLITE_OK)
446 return false;
447
448 res = logged_sqlite3_exec(sql, "INSERT INTO sequence_numbers "
449 "SELECT "
450 "(url || '/' || groupname), "
451 "major, "
452 "minor "
453 "FROM tmp", NULL, NULL, errmsg);
454 if (res != SQLITE_OK)
455 return false;
456
457 res = logged_sqlite3_exec(sql, "DROP TABLE tmp", NULL, NULL, errmsg);
458 if (res != SQLITE_OK)
459 return false;
460
461
462 // migrate the netserver_manifests table
463 if (!move_table(sql, errmsg,
464 "netserver_manifests",
465 "tmp",
466 "("
467 "url not null,"
468 "groupname not null,"
469 "manifest not null,"
470 "unique(url, groupname, manifest)"
471 ")"
472 ))
473 return false;
474
475 res = logged_sqlite3_exec(sql, "CREATE TABLE netserver_manifests "
476 "("
477 "url not null, -- url of some server\n"
478 "manifest not null, -- manifest which exists on url\n"
479 "unique(url, manifest)"
480 ")", NULL, NULL, errmsg);
481 if (res != SQLITE_OK)
482 return false;
483
484 res = logged_sqlite3_exec(sql, "INSERT INTO netserver_manifests "
485 "SELECT "
486 "(url || '/' || groupname), "
487 "manifest "
488 "FROM tmp", NULL, NULL, errmsg);
489 if (res != SQLITE_OK)
490 return false;
491
492 res = logged_sqlite3_exec(sql, "DROP TABLE tmp", NULL, NULL, errmsg);
493 if (res != SQLITE_OK)
494 return false;
495
496 return true;
497}
498
499static bool
500migrate_client_add_hashes_and_merkle_trees(sqlite3 * sql,
501 char ** errmsg,
502 app_state *)
503{
504
505 // add the column to manifest_certs
506 if (!move_table(sql, errmsg,
507 "manifest_certs",
508 "tmp",
509 "("
510 "id not null,"
511 "name not null,"
512 "value not null,"
513 "keypair not null,"
514 "signature not null,"
515 "unique(name, id, value, keypair, signature)"
516 ")"))
517 return false;
518
519 int res = logged_sqlite3_exec(sql, "CREATE TABLE manifest_certs\n"
520 "(\n"
521 "hash not null unique, -- hash of remaining fields separated by \":\"\n"
522 "id not null, -- joins with manifests.id or manifest_deltas.id\n"
523 "name not null, -- opaque string chosen by user\n"
524 "value not null, -- opaque blob\n"
525 "keypair not null, -- joins with public_keys.id\n"
526 "signature not null, -- RSA/SHA1 signature of \"[name@id:val]\"\n"
527 "unique(name, id, value, keypair, signature)\n"
528 ")", NULL, NULL, errmsg);
529 if (res != SQLITE_OK)
530 return false;
531
532 res = logged_sqlite3_exec(sql, "INSERT INTO manifest_certs "
533 "SELECT "
534 "sha1(':', id, name, value, keypair, signature), "
535 "id, name, value, keypair, signature "
536 "FROM tmp", NULL, NULL, errmsg);
537 if (res != SQLITE_OK)
538 return false;
539
540 res = logged_sqlite3_exec(sql, "DROP TABLE tmp", NULL, NULL, errmsg);
541 if (res != SQLITE_OK)
542 return false;
543
544 // add the column to file_certs
545 if (!move_table(sql, errmsg,
546 "file_certs",
547 "tmp",
548 "("
549 "id not null,"
550 "name not null,"
551 "value not null,"
552 "keypair not null,"
553 "signature not null,"
554 "unique(name, id, value, keypair, signature)"
555 ")"))
556 return false;
557
558 res = logged_sqlite3_exec(sql, "CREATE TABLE file_certs\n"
559 "(\n"
560 "hash not null unique, -- hash of remaining fields separated by \":\"\n"
561 "id not null, -- joins with files.id or file_deltas.id\n"
562 "name not null, -- opaque string chosen by user\n"
563 "value not null, -- opaque blob\n"
564 "keypair not null, -- joins with public_keys.id\n"
565 "signature not null, -- RSA/SHA1 signature of \"[name@id:val]\"\n"
566 "unique(name, id, value, keypair, signature)\n"
567 ")", NULL, NULL, errmsg);
568 if (res != SQLITE_OK)
569 return false;
570
571 res = logged_sqlite3_exec(sql, "INSERT INTO file_certs "
572 "SELECT "
573 "sha1(':', id, name, value, keypair, signature), "
574 "id, name, value, keypair, signature "
575 "FROM tmp", NULL, NULL, errmsg);
576 if (res != SQLITE_OK)
577 return false;
578
579 res = logged_sqlite3_exec(sql, "DROP TABLE tmp", NULL, NULL, errmsg);
580 if (res != SQLITE_OK)
581 return false;
582
583 // add the column to public_keys
584 if (!move_table(sql, errmsg,
585 "public_keys",
586 "tmp",
587 "("
588 "id primary key,"
589 "keydata not null"
590 ")"))
591 return false;
592
593 res = logged_sqlite3_exec(sql, "CREATE TABLE public_keys\n"
594 "(\n"
595 "hash not null unique, -- hash of remaining fields separated by \":\"\n"
596 "id primary key, -- key identifier chosen by user\n"
597 "keydata not null -- RSA public params\n"
598 ")", NULL, NULL, errmsg);
599 if (res != SQLITE_OK)
600 return false;
601
602 res = logged_sqlite3_exec(sql, "INSERT INTO public_keys "
603 "SELECT "
604 "sha1(':', id, keydata), "
605 "id, keydata "
606 "FROM tmp", NULL, NULL, errmsg);
607 if (res != SQLITE_OK)
608 return false;
609
610 res = logged_sqlite3_exec(sql, "DROP TABLE tmp", NULL, NULL, errmsg);
611 if (res != SQLITE_OK)
612 return false;
613
614 // add the column to private_keys
615 if (!move_table(sql, errmsg,
616 "private_keys",
617 "tmp",
618 "("
619 "id primary key,"
620 "keydata not null"
621 ")"))
622 return false;
623
624 res = logged_sqlite3_exec(sql, "CREATE TABLE private_keys\n"
625 "(\n"
626 "hash not null unique, -- hash of remaining fields separated by \":\"\n"
627 "id primary key, -- as in public_keys (same identifiers, in fact)\n"
628 "keydata not null -- encrypted RSA private params\n"
629 ")", NULL, NULL, errmsg);
630 if (res != SQLITE_OK)
631 return false;
632
633 res = logged_sqlite3_exec(sql, "INSERT INTO private_keys "
634 "SELECT "
635 "sha1(':', id, keydata), "
636 "id, keydata "
637 "FROM tmp", NULL, NULL, errmsg);
638 if (res != SQLITE_OK)
639 return false;
640
641 res = logged_sqlite3_exec(sql, "DROP TABLE tmp", NULL, NULL, errmsg);
642 if (res != SQLITE_OK)
643 return false;
644
645 // add the merkle tree stuff
646
647 res = logged_sqlite3_exec(sql,
648 "CREATE TABLE merkle_nodes\n"
649 "(\n"
650 "type not null, -- \"key\", \"mcert\", \"fcert\", \"manifest\"\n"
651 "collection not null, -- name chosen by user\n"
652 "level not null, -- tree level this prefix encodes\n"
653 "prefix not null, -- label identifying node in tree\n"
654 "body not null, -- binary, base64'ed node contents\n"
655 "unique(type, collection, level, prefix)\n"
656 ")", NULL, NULL, errmsg);
657 if (res != SQLITE_OK)
658 return false;
659
660 return true;
661}
662
663static bool
664migrate_client_to_revisions(sqlite3 * sql,
665 char ** errmsg,
666 app_state *)
667{
668 int res;
669
670 res = logged_sqlite3_exec(sql, "DROP TABLE schema_version;", NULL, NULL, errmsg);
671 if (res != SQLITE_OK)
672 return false;
673
674 res = logged_sqlite3_exec(sql, "DROP TABLE posting_queue;", NULL, NULL, errmsg);
675 if (res != SQLITE_OK)
676 return false;
677
678 res = logged_sqlite3_exec(sql, "DROP TABLE incoming_queue;", NULL, NULL, errmsg);
679 if (res != SQLITE_OK)
680 return false;
681
682 res = logged_sqlite3_exec(sql, "DROP TABLE sequence_numbers;", NULL, NULL, errmsg);
683 if (res != SQLITE_OK)
684 return false;
685
686 res = logged_sqlite3_exec(sql, "DROP TABLE file_certs;", NULL, NULL, errmsg);
687 if (res != SQLITE_OK)
688 return false;
689
690 res = logged_sqlite3_exec(sql, "DROP TABLE netserver_manifests;", NULL, NULL, errmsg);
691 if (res != SQLITE_OK)
692 return false;
693
694 res = logged_sqlite3_exec(sql, "DROP TABLE merkle_nodes;", NULL, NULL, errmsg);
695 if (res != SQLITE_OK)
696 return false;
697
698 res = logged_sqlite3_exec(sql,
699 "CREATE TABLE merkle_nodes\n"
700 "(\n"
701 "type not null, -- \"key\", \"mcert\", \"fcert\", \"rcert\"\n"
702 "collection not null, -- name chosen by user\n"
703 "level not null, -- tree level this prefix encodes\n"
704 "prefix not null, -- label identifying node in tree\n"
705 "body not null, -- binary, base64'ed node contents\n"
706 "unique(type, collection, level, prefix)\n"
707 ")", NULL, NULL, errmsg);
708 if (res != SQLITE_OK)
709 return false;
710
711 res = logged_sqlite3_exec(sql, "CREATE TABLE revision_certs\n"
712 "(\n"
713 "hash not null unique, -- hash of remaining fields separated by \":\"\n"
714 "id not null, -- joins with revisions.id\n"
715 "name not null, -- opaque string chosen by user\n"
716 "value not null, -- opaque blob\n"
717 "keypair not null, -- joins with public_keys.id\n"
718 "signature not null, -- RSA/SHA1 signature of \"[name@id:val]\"\n"
719 "unique(name, id, value, keypair, signature)\n"
720 ")", NULL, NULL, errmsg);
721 if (res != SQLITE_OK)
722 return false;
723
724 res = logged_sqlite3_exec(sql, "CREATE TABLE revisions\n"
725 "(\n"
726 "id primary key, -- SHA1(text of revision)\n"
727 "data not null -- compressed, encoded contents of a revision\n"
728 ")", NULL, NULL, errmsg);
729 if (res != SQLITE_OK)
730 return false;
731
732 res = logged_sqlite3_exec(sql, "CREATE TABLE revision_ancestry\n"
733 "(\n"
734 "parent not null, -- joins with revisions.id\n"
735 "child not null, -- joins with revisions.id\n"
736 "unique(parent, child)\n"
737 ")", NULL, NULL, errmsg);
738 if (res != SQLITE_OK)
739 return false;
740
741 return true;
742}
743
744
745static bool
746migrate_client_to_epochs(sqlite3 * sql,
747 char ** errmsg,
748 app_state *)
749{
750 int res;
751
752 res = logged_sqlite3_exec(sql, "DROP TABLE merkle_nodes;", NULL, NULL, errmsg);
753 if (res != SQLITE_OK)
754 return false;
755
756
757 res = logged_sqlite3_exec(sql,
758 "CREATE TABLE branch_epochs\n"
759 "(\n"
760 "hash not null unique, -- hash of remaining fields separated by \":\"\n"
761 "branch not null unique, -- joins with revision_certs.value\n"
762 "epoch not null -- random hex-encoded id\n"
763 ");", NULL, NULL, errmsg);
764 if (res != SQLITE_OK)
765 return false;
766
767 return true;
768}
769
770static bool
771migrate_client_to_vars(sqlite3 * sql,
772 char ** errmsg,
773 app_state *)
774{
775 int res;
776
777 res = logged_sqlite3_exec(sql,
778 "CREATE TABLE db_vars\n"
779 "(\n"
780 "domain not null, -- scope of application of a var\n"
781 "name not null, -- var key\n"
782 "value not null, -- var value\n"
783 "unique(domain, name)\n"
784 ");", NULL, NULL, errmsg);
785 if (res != SQLITE_OK)
786 return false;
787
788 return true;
789}
790
791static bool
792migrate_client_to_add_indexes(sqlite3 * sql,
793 char ** errmsg,
794 app_state *)
795{
796 int res;
797
798 res = logged_sqlite3_exec(sql,
799 "CREATE INDEX revision_ancestry__child "
800 "ON revision_ancestry (child)",
801 NULL, NULL, errmsg);
802 if (res != SQLITE_OK)
803 return false;
804
805 res = logged_sqlite3_exec(sql,
806 "CREATE INDEX revision_certs__id "
807 "ON revision_certs (id);",
808 NULL, NULL, errmsg);
809 if (res != SQLITE_OK)
810 return false;
811
812 res = logged_sqlite3_exec(sql,
813 "CREATE INDEX revision_certs__name_value "
814 "ON revision_certs (name, value);",
815 NULL, NULL, errmsg);
816 if (res != SQLITE_OK)
817 return false;
818
819 return true;
820}
821
822static int
823extract_key(void *ptr, int ncols, char **values, char **names)
824{
825 // This is stupid. The cast should not be needed.
826 map<string, string> *out = (map<string, string>*)ptr;
827 I(ncols == 2);
828 out->insert(make_pair(string(values[0]), string(values[1])));
829 return 0;
830}
831static bool
832migrate_client_to_external_privkeys(sqlite3 * sql,
833 char ** errmsg,
834 app_state *app)
835{
836 int res;
837 map<string, string> pub, priv;
838 vector<keypair> pairs;
839
840 res = logged_sqlite3_exec(sql,
841 "SELECT id, keydata FROM private_keys;",
842 &extract_key, &priv, errmsg);
843 if (res != SQLITE_OK)
844 return false;
845
846 res = logged_sqlite3_exec(sql,
847 "SELECT id, keydata FROM public_keys;",
848 &extract_key, &pub, errmsg);
849 if (res != SQLITE_OK)
850 return false;
851
852 for (map<string, string>::const_iterator i = priv.begin();
853 i != priv.end(); ++i)
854 {
855 rsa_keypair_id ident = i->first;
856 base64< arc4<rsa_priv_key> > old_priv = i->second;
857 map<string, string>::const_iterator j = pub.find(i->first);
858 keypair kp;
859 migrate_private_key(*app, ident, old_priv, kp);
860 MM(kp.pub);
861 if (j != pub.end())
862 {
863 base64< rsa_pub_key > pub = j->second;
864 MM(pub);
865 N(keys_match(ident, pub, ident, kp.pub),
866 F("public and private keys for %s don't match") % ident);
867 }
868
869 P(F("moving key '%s' from database to %s")
870 % ident % app->keys.get_key_dir());
871 app->keys.put_key_pair(ident, kp);
872 }
873
874 res = logged_sqlite3_exec(sql, "DROP TABLE private_keys;", NULL, NULL, errmsg);
875 if (res != SQLITE_OK)
876 return false;
877
878 return true;
879}
880
881static bool
882migrate_client_to_add_rosters(sqlite3 * sql,
883 char ** errmsg,
884 app_state *)
885{
886 int res;
887
888 res = logged_sqlite3_exec(sql,
889 "CREATE TABLE rosters\n"
890 "(\n"
891 "id primary key, -- strong hash of the roster\n"
892 "data not null -- compressed, encoded contents of the roster\n"
893 ");",
894 NULL, NULL, errmsg);
895 if (res != SQLITE_OK)
896 return false;
897
898 res = logged_sqlite3_exec(sql,
899 "CREATE TABLE roster_deltas\n"
900 "(\n"
901 "id not null, -- strong hash of the roster\n"
902 "base not null, -- joins with either rosters.id or roster_deltas.id\n"
903 "delta not null, -- rdiff to construct current from base\n"
904 "unique(id, base)\n"
905 ");",
906 NULL, NULL, errmsg);
907 if (res != SQLITE_OK)
908 return false;
909
910 res = logged_sqlite3_exec(sql,
911 "CREATE TABLE revision_roster\n"
912 "(\n"
913 "rev_id primary key, -- joins with revisions.id\n"
914 "roster_id not null -- joins with either rosters.id or roster_deltas.id\n"
915 ");",
916 NULL, NULL, errmsg);
917 if (res != SQLITE_OK)
918 return false;
919
920 res = logged_sqlite3_exec(sql,
921 "CREATE TABLE next_roster_node_number\n"
922 "(\n"
923 "node primary key -- only one entry in this table, ever\n"
924 ");",
925 NULL, NULL, errmsg);
926 if (res != SQLITE_OK)
927 return false;
928
929 return true;
930}
931
932static void
933sqlite3_unbase64_fn(sqlite3_context *f, int nargs, sqlite3_value ** args)
934{
935 if (nargs != 1)
936 {
937 sqlite3_result_error(f, "need exactly 1 arg to unbase64()", -1);
938 return;
939 }
940 data decoded;
941 decode_base64(base64<data>(string(reinterpret_cast<char const*>(sqlite3_value_text(args[0])))), decoded);
942 sqlite3_result_blob(f, decoded().c_str(), decoded().size(), SQLITE_TRANSIENT);
943}
944
945// I wish I had a form of ALTER TABLE COMMENT on sqlite3
946static bool
947migrate_files_BLOB(sqlite3 * sql,
948 char ** errmsg,
949 app_state *app)
950{
951 int res;
952 I(sqlite3_create_function(sql, "unbase64", -1,
953 SQLITE_UTF8, NULL,
954 &sqlite3_unbase64_fn,
955 NULL, NULL) == 0);
956 // change the encoding of file(_delta)s
957 if (!move_table(sql, errmsg,
958 "files",
959 "tmp",
960 "("
961 "id primary key,"
962 "data not null"
963 ")"))
964 return false;
965
966 res = logged_sqlite3_exec(sql, "CREATE TABLE files\n"
967 "\t(\n"
968 "\tid primary key, -- strong hash of file contents\n"
969 "\tdata not null -- compressed contents of a file\n"
970 "\t)", NULL, NULL, errmsg);
971 if (res != SQLITE_OK)
972 return false;
973
974 res = logged_sqlite3_exec(sql, "INSERT INTO files "
975 "SELECT id, unbase64(data) "
976 "FROM tmp", NULL, NULL, errmsg);
977 if (res != SQLITE_OK)
978 return false;
979
980 res = logged_sqlite3_exec(sql, "DROP TABLE tmp", NULL, NULL, errmsg);
981 if (res != SQLITE_OK)
982 return false;
983
984 if (!move_table(sql, errmsg,
985 "file_deltas",
986 "tmp",
987 "("
988 "id not null,"
989 "base not null,"
990 "delta not null"
991 ")"))
992 return false;
993
994 res = logged_sqlite3_exec(sql, "CREATE TABLE file_deltas\n"
995 "\t(\n"
996 "\tid not null, -- strong hash of file contents\n"
997 "\tbase not null, -- joins with files.id or file_deltas.id\n"
998 "\tdelta not null, -- compressed rdiff to construct current from base\n"
999 "\tunique(id, base)\n"
1000 "\t)", NULL, NULL, errmsg);
1001 if (res != SQLITE_OK)
1002 return false;
1003
1004 res = logged_sqlite3_exec(sql, "INSERT INTO file_deltas "
1005 "SELECT id, base, unbase64(delta) "
1006 "FROM tmp", NULL, NULL, errmsg);
1007 if (res != SQLITE_OK)
1008 return false;
1009
1010 res = logged_sqlite3_exec(sql, "DROP TABLE tmp", NULL, NULL, errmsg);
1011 if (res != SQLITE_OK)
1012 return false;
1013
1014 // migrate other contents which are accessed by get|put_version
1015 res = logged_sqlite3_exec(sql, "UPDATE manifests SET data=unbase64(data)",
1016 NULL, NULL, errmsg);
1017 if (res != SQLITE_OK)
1018 return false;
1019 res = logged_sqlite3_exec(sql, "UPDATE manifest_deltas "
1020 "SET delta=unbase64(delta)", NULL, NULL, errmsg);
1021 if (res != SQLITE_OK)
1022 return false;
1023 res = logged_sqlite3_exec(sql, "UPDATE rosters SET data=unbase64(data) ",
1024 NULL, NULL, errmsg);
1025 if (res != SQLITE_OK)
1026 return false;
1027 res = logged_sqlite3_exec(sql, "UPDATE roster_deltas "
1028 "SET delta=unbase64(delta)", NULL, NULL, errmsg);
1029 if (res != SQLITE_OK)
1030 return false;
1031
1032 res = logged_sqlite3_exec(sql, "UPDATE db_vars "
1033 "SET value=unbase64(value),name=unbase64(name)", NULL, NULL, errmsg);
1034 if (res != SQLITE_OK)
1035 return false;
1036 res = logged_sqlite3_exec(sql, "UPDATE public_keys "
1037 "SET keydata=unbase64(keydata)", NULL, NULL, errmsg);
1038 if (res != SQLITE_OK)
1039 return false;
1040 res = logged_sqlite3_exec(sql, "UPDATE revision_certs "
1041 "SET value=unbase64(value),signature=unbase64(signature)",
1042 NULL, NULL, errmsg);
1043 if (res != SQLITE_OK)
1044 return false;
1045 res = logged_sqlite3_exec(sql, "UPDATE manifest_certs "
1046 "SET value=unbase64(value),signature=unbase64(signature) ",
1047 NULL, NULL, errmsg);
1048 if (res != SQLITE_OK)
1049 return false;
1050 res = logged_sqlite3_exec(sql, "UPDATE revisions "
1051 "SET data=unbase64(data)", NULL, NULL, errmsg);
1052 if (res != SQLITE_OK)
1053 return false;
1054 res = logged_sqlite3_exec(sql, "UPDATE branch_epochs "
1055 "SET branch=unbase64(branch)", NULL, NULL, errmsg);
1056 if (res != SQLITE_OK)
1057 return false;
1058 return true;
1059}
1060
1061void
1062migrate_monotone_schema(sqlite3 *sql, app_state *app)
1063{
1064
1065 migrator m;
1066 m.set_app(app);
1067
1068 m.add("edb5fa6cef65bcb7d0c612023d267c3aeaa1e57a",
1069 &migrate_client_merge_url_and_group);
1070
1071 m.add("f042f3c4d0a4f98f6658cbaf603d376acf88ff4b",
1072 &migrate_client_add_hashes_and_merkle_trees);
1073
1074 m.add("8929e54f40bf4d3b4aea8b037d2c9263e82abdf4",
1075 &migrate_client_to_revisions);
1076
1077 m.add("c1e86588e11ad07fa53e5d294edc043ce1d4005a",
1078 &migrate_client_to_epochs);
1079
1080 m.add("40369a7bda66463c5785d160819ab6398b9d44f4",
1081 &migrate_client_to_vars);
1082
1083 m.add("e372b508bea9b991816d1c74680f7ae10d2a6d94",
1084 &migrate_client_to_add_indexes);
1085
1086 m.add("1509fd75019aebef5ac3da3a5edf1312393b70e9",
1087 &migrate_client_to_external_privkeys);
1088
1089 m.add("bd86f9a90b5d552f0be1fa9aee847ea0f317778b",
1090 &migrate_client_to_add_rosters);
1091
1092 m.add("1db80c7cee8fa966913db1a463ed50bf1b0e5b0e",
1093 &migrate_files_BLOB);
1094
1095 // IMPORTANT: whenever you modify this to add a new schema version, you must
1096 // also add a new migration test for the new schema version. See
1097 // tests/t_migrate_schema.at for details.
1098
1099 m.migrate(sql, "9d2b5d7b86df00c30ac34fe87a3c20f1195bb2df");
1100}
1101
1102// Local Variables:
1103// mode: C++
1104// fill-column: 76
1105// c-file-style: "gnu"
1106// indent-tabs-mode: nil
1107// End:
1108// vim: et:sw=2:sts=2:ts=2:cino=>2s,{s,\:s,+s,t0,g0,^-2,e-2,n-2,p2s,(0,=s:

Archive Download this file

Branches

Tags

Quick Links:     www.monotone.ca    -     Downloads    -     Documentation    -     Wiki    -     Code Forge    -     Build Status