monotone

monotone Mtn Source Tree

Root/schema_migration.cc

1// Copyright (C) 2002 Graydon Hoare <graydon@pobox.com>
2//
3// This program is made available under the GNU GPL version 2.0 or
4// greater. See the accompanying file COPYING for details.
5//
6// This program is distributed WITHOUT ANY WARRANTY; without even the
7// implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
8// PURPOSE.
9
10#include "base.hh"
11#include <boost/tokenizer.hpp>
12#include "lexical_cast.hh"
13#include "sqlite/sqlite3.h"
14#include <cstring>
15
16#include "sanity.hh"
17#include "schema_migration.hh"
18#include "key_store.hh"
19#include "transforms.hh"
20#include "ui.hh"
21#include "constants.hh"
22
23using std::string;
24
25// this file knows how to migrate schema databases. the general strategy is
26// to hash each schema we ever use, and make a list of the SQL commands
27// required to get from one hash value to the next. when you do a
28// migration, the migrator locates your current db's state on the list and
29// then runs all the migration functions between that point and the target
30// of the migration.
31
32// you will notice a little bit of duplicated code between here and
33// database.cc; this was originally to facilitate inclusion of migration
34// capability into the depot code, but is now preserved because the code
35// in this file is easier to write and understand if it speaks directly
36// to sqlite.
37
38// Wrappers around the bare sqlite3 API. We do not use sqlite3_exec because
39// we want the better error handling that sqlite3_prepare_v2 gives us.
40
41void
42assert_sqlite3_ok(sqlite3 * db)
43{
44 int errcode = sqlite3_errcode(db);
45
46 if (errcode == SQLITE_OK)
47 return;
48
49 char const * errmsg = sqlite3_errmsg(db);
50
51 // first log the code so we can find _out_ what the confusing code
52 // was... note that code does not uniquely identify the errmsg, unlike
53 // errno's.
54 L(FL("sqlite error: %d: %s") % errcode % errmsg);
55
56 // Check the string to see if it looks like an informative_failure
57 // thrown from within an SQL extension function, caught, and turned
58 // into a call to sqlite3_result_error. (Extension functions have to
59 // do this to avoid corrupting sqlite's internal state.) If it is,
60 // rethrow it rather than feeding it to E(), lest we get "error:
61 // sqlite error: error: " ugliness.
62 char const *pfx = _("error: ");
63 if (!std::strncmp(errmsg, pfx, strlen(pfx)))
64 throw informative_failure(errmsg);
65
66 // sometimes sqlite is not very helpful
67 // so we keep a table of errors people have gotten and more helpful versions
68 char const * auxiliary_message = "";
69 switch (errcode)
70 {
71 // All memory-exhaustion conditions should give the same diagnostic.
72 case SQLITE_NOMEM:
73 throw std::bad_alloc();
74
75 // These diagnostics generally indicate an operating-system-level
76 // failure. It would be nice to throw strerror(errno) in there but
77 // we cannot assume errno is still valid by the time we get here.
78 case SQLITE_IOERR:
79 case SQLITE_CANTOPEN:
80 case SQLITE_PROTOCOL:
81 auxiliary_message
82 = _("make sure database and containing directory are writeable\n"
83 "and you have not run out of disk space");
84 break;
85
86 // These error codes may indicate someone is trying to load a database
87 // so old that it's in sqlite 2's disk format (monotone 0.16 or
88 // older).
89 case SQLITE_CORRUPT:
90 case SQLITE_NOTADB:
91 auxiliary_message
92 = _("(if this is a database last used by monotone 0.16 or older,\n"
93 "you must follow a special procedure to make it usable again.\n"
94 "see the file UPGRADE, in the distribution, for instructions.)");
95
96 default:
97 break;
98 }
99
100 // if the auxiliary message is empty, the \n will be stripped off too
101 E(false, F("sqlite error: %s\n%s") % errmsg % auxiliary_message);
102}
103
104
105namespace
106{
107 struct sql
108 {
109 sql(sqlite3 * db, int cols, char const *cmd, char const **afterp = 0)
110 : stmt(0), ncols(cols)
111 {
112 sqlite3_stmt * s;
113
114 char const * after;
115 L(FL("executing SQL '%s'") % cmd);
116
117 sqlite3_prepare_v2(db, cmd, strlen(cmd), &s, &after);
118 assert_sqlite3_ok(db);
119
120 I(s);
121 if (afterp)
122 *afterp = after;
123 else
124 I(*after == 0);
125 I(sqlite3_column_count(s) == ncols);
126 stmt = s;
127 }
128 ~sql()
129 {
130 if (stmt)
131 sqlite3_finalize(stmt);
132 }
133
134 bool step()
135 {
136 int res = sqlite3_step(stmt);
137 if (res == SQLITE_ROW)
138 return true;
139 if (res == SQLITE_DONE)
140 {
141 L(FL("success"));
142 return false;
143 }
144 // Diagnostics from sqlite3_result_error show up in sqlite3_errmsg
145 // only after sqlite3_finalize or sqlite3_reset are called on the
146 // stmt object. See SQLite ticket #1640.
147 sqlite3 * db = sqlite3_db_handle(stmt);
148 sqlite3_finalize(stmt);
149 stmt = 0;
150 assert_sqlite3_ok(db);
151 I(false);
152 }
153 int column_int(int col)
154 {
155 I(col >= 0 && col < ncols);
156 return sqlite3_column_int(stmt, col);
157 }
158 string column_string(int col)
159 {
160 I(col >= 0 && col < ncols);
161 return string(reinterpret_cast<char const *>
162 (sqlite3_column_text(stmt, col)));
163 }
164 bool column_nonnull(int col)
165 {
166 I(col >= 0 && col < ncols);
167 return sqlite3_column_type(stmt, col) != SQLITE_NULL;
168 }
169
170 // convenience for executing a sequence of sql statements,
171 // none of which returns any rows.
172 static void exec(sqlite3 * db, char const * cmd)
173 {
174 do
175 {
176 sql stmt(db, 0, cmd, &cmd);
177 I(stmt.step() == false);
178 }
179 while (*cmd != '\0');
180 }
181
182 // convenience for evaluating an expression that returns a single number.
183 static int value(sqlite3 * db, char const * cmd)
184 {
185 sql stmt(db, 1, cmd);
186
187 I(stmt.step() == true);
188 int res = stmt.column_int(0);
189 I(stmt.step() == false);
190
191 return res;
192 }
193
194 // convenience for making functions
195 static void create_function(sqlite3 * db, char const * name,
196 void (*fn)(sqlite3_context *,
197 int, sqlite3_value **))
198 {
199 sqlite3_create_function(db, name, -1, SQLITE_UTF8, 0, fn, 0, 0);
200 assert_sqlite3_ok(db);
201 }
202
203 private:
204 sqlite3_stmt * stmt;
205 int ncols;
206 };
207
208 struct transaction
209 {
210 transaction(sqlite3 * s) : db(s), committed(false)
211 {
212 sql::exec(db, "BEGIN EXCLUSIVE");
213 }
214 void commit()
215 {
216 I(committed == false);
217 committed = true;
218 }
219 ~transaction()
220 {
221 if (committed)
222 sql::exec(db, "COMMIT");
223 else
224 sql::exec(db, "ROLLBACK");
225 }
226 private:
227 sqlite3 * db;
228 bool committed;
229 };
230}
231
232// SQL extension functions.
233
234// sqlite3_value_text returns unsigned char const *, which is inconvenient
235inline char const *
236sqlite3_value_cstr(sqlite3_value * arg)
237{
238 return reinterpret_cast<char const *>(sqlite3_value_text(arg));
239}
240
241inline bool is_ws(char c)
242{
243 return c == '\r' || c == '\n' || c == '\t' || c == ' ';
244}
245
246static void
247sqlite_sha1_fn(sqlite3_context *f, int nargs, sqlite3_value ** args)
248{
249 if (nargs <= 1)
250 {
251 sqlite3_result_error(f, "need at least 1 arg to sha1()", -1);
252 return;
253 }
254
255 string tmp;
256 if (nargs == 1)
257 {
258 char const * s = sqlite3_value_cstr(args[0]);
259 char const * end = s + sqlite3_value_bytes(args[0]) - 1;
260 remove_copy_if(s, end, back_inserter(tmp), is_ws);
261 }
262 else
263 {
264 char const * sep = sqlite3_value_cstr(args[0]);
265
266 for (int i = 1; i < nargs; ++i)
267 {
268 if (i > 1)
269 tmp += sep;
270 char const * s = sqlite3_value_cstr(args[i]);
271 char const * end = s + sqlite3_value_bytes(args[i]) - 1;
272 remove_copy_if(s, end, back_inserter(tmp), is_ws);
273 }
274 }
275
276 id hash;
277 calculate_ident(data(tmp), hash);
278 sqlite3_result_blob(f, hash().c_str(), hash().size(), SQLITE_TRANSIENT);
279}
280
281static void
282sqlite3_unbase64_fn(sqlite3_context *f, int nargs, sqlite3_value ** args)
283{
284 if (nargs != 1)
285 {
286 sqlite3_result_error(f, "need exactly 1 arg to unbase64()", -1);
287 return;
288 }
289 string decoded;
290
291 // This operation may throw informative_failure. We must intercept that
292 // and turn it into a call to sqlite3_result_error, or rollback will fail.
293 try
294 {
295 decoded = decode_base64_as<string>(sqlite3_value_cstr(args[0]));
296 }
297 catch (informative_failure & e)
298 {
299 sqlite3_result_error(f, e.what(), -1);
300 return;
301 }
302 sqlite3_result_blob(f, decoded.c_str(), decoded.size(), SQLITE_TRANSIENT);
303}
304
305static void
306sqlite3_unhex_fn(sqlite3_context *f, int nargs, sqlite3_value **args)
307{
308 if (nargs != 1)
309 {
310 sqlite3_result_error(f, "need exactly 1 arg to unhex()", -1);
311 return;
312 }
313 string decoded;
314
315 // This operation may throw informative_failure. We must intercept that
316 // and turn it into a call to sqlite3_result_error, or rollback will fail.
317 try
318 {
319 decoded = decode_hexenc(sqlite3_value_cstr(args[0]));
320 }
321 catch (informative_failure & e)
322 {
323 sqlite3_result_error(f, e.what(), -1);
324 return;
325 }
326 // This is only ever used with 20-byte SHA1 hashes or empty strings, so
327 // make sure that's what we've got.
328 if (decoded.size() != constants::idlen_bytes && decoded.size() != 0)
329 {
330 sqlite3_result_error(f, "unhex() result is the wrong length", -1);
331 return;
332 }
333
334 sqlite3_result_blob(f, decoded.data(), decoded.size(), SQLITE_TRANSIENT);
335}
336
337// Here are all of the migration steps. Almost all of them can be expressed
338// entirely as a series of SQL statements; those statements are packaged
339// into a long, continued string constant for the step. A few require a
340// function instead.
341//
342// Please keep this list in the same order as the migration_events table
343// below.
344
345char const migrate_merge_url_and_group[] =
346 // migrate the posting_queue table
347 "ALTER TABLE posting_queue RENAME TO tmp;"
348 "CREATE TABLE posting_queue"
349 " ( url not null, -- URL we are going to send this to\n"
350 " content not null -- the packets we're going to send\n"
351 " );"
352 "INSERT INTO posting_queue"
353 " SELECT (url || '/' || groupname), content FROM tmp;"
354 "DROP TABLE tmp;"
355
356 // migrate the incoming_queue table
357 "ALTER TABLE incoming_queue RENAME TO tmp;"
358 "CREATE TABLE incoming_queue "
359 " ( url not null, -- URL we got this bundle from\n"
360 " content not null -- the packets we're going to read\n"
361 " );"
362 "INSERT INTO incoming_queue"
363 " SELECT (url || '/' || groupname), content FROM tmp;"
364 "DROP TABLE tmp;"
365
366 // migrate the sequence_numbers table
367 "ALTER TABLE sequence_numbers RENAME TO tmp;"
368 "CREATE TABLE sequence_numbers "
369 " ( url primary key, -- URL to read from\n"
370 " major not null, -- 0 in news servers, may be higher in depots\n"
371 " minor not null -- last article / packet sequence number we got\n"
372 " );"
373 "INSERT INTO sequence_numbers"
374 " SELECT (url || '/' || groupname), major, minor FROM tmp;"
375 "DROP TABLE tmp;"
376
377 // migrate the netserver_manifests table
378 "ALTER TABLE netserver_manifests RENAME TO tmp;"
379 "CREATE TABLE netserver_manifests"
380 " ( url not null, -- url of some server\n"
381 " manifest not null, -- manifest which exists on url\n"
382 " unique(url, manifest)"
383 " );"
384 "INSERT INTO netserver_manifests"
385 " SELECT (url || '/' || groupname), manifest FROM tmp;"
386
387 "DROP TABLE tmp;"
388 ;
389
390char const migrate_add_hashes_and_merkle_trees[] =
391 // add the column to manifest_certs
392 "ALTER TABLE manifest_certs RENAME TO tmp;"
393 "CREATE TABLE manifest_certs"
394 " ( hash not null unique, -- hash of remaining fields separated by \":\"\n"
395 " id not null, -- joins with manifests.id or manifest_deltas.id\n"
396 " name not null, -- opaque string chosen by user\n"
397 " value not null, -- opaque blob\n"
398 " keypair not null, -- joins with public_keys.id\n"
399 " signature not null, -- RSA/SHA1 signature of \"[name@id:val]\"\n"
400 " unique(name, id, value, keypair, signature)"
401 " );"
402 "INSERT INTO manifest_certs"
403 " SELECT sha1(':', id, name, value, keypair, signature),"
404 " id, name, value, keypair, signature"
405 " FROM tmp;"
406 "DROP TABLE tmp;"
407
408 // add the column to file_certs
409 "ALTER TABLE file_certs RENAME TO tmp;"
410 "CREATE TABLE file_certs"
411 " ( hash not null unique, -- hash of remaining fields separated by \":\"\n"
412 " id not null, -- joins with files.id or file_deltas.id\n"
413 " name not null, -- opaque string chosen by user\n"
414 " value not null, -- opaque blob\n"
415 " keypair not null, -- joins with public_keys.id\n"
416 " signature not null, -- RSA/SHA1 signature of \"[name@id:val]\"\n"
417 " unique(name, id, value, keypair, signature)"
418 " );"
419 "INSERT INTO file_certs"
420 " SELECT sha1(':', id, name, value, keypair, signature),"
421 " id, name, value, keypair, signature"
422 " FROM tmp;"
423 "DROP TABLE tmp;"
424
425 // add the column to public_keys
426 "ALTER TABLE public_keys RENAME TO tmp;"
427 "CREATE TABLE public_keys"
428 " ( hash not null unique, -- hash of remaining fields separated by \":\"\n"
429 " id primary key, -- key identifier chosen by user\n"
430 " keydata not null -- RSA public params\n"
431 " );"
432 "INSERT INTO public_keys SELECT sha1(':',id,keydata), id, keydata FROM tmp;"
433 "DROP TABLE tmp;"
434
435 // add the column to private_keys
436 "ALTER TABLE private_keys RENAME TO tmp;"
437 "CREATE TABLE private_keys"
438 " ( hash not null unique, -- hash of remaining fields separated by \":\"\n"
439 " id primary key, -- as in public_keys (same identifiers, in fact)\n"
440 " keydata not null -- encrypted RSA private params\n"
441 " );"
442 "INSERT INTO private_keys SELECT sha1(':',id,keydata), id, keydata FROM tmp;"
443 "DROP TABLE tmp;"
444
445 // add the merkle tree stuff
446 "CREATE TABLE merkle_nodes"
447 " ( type not null, -- \"key\", \"mcert\", \"fcert\", \"manifest\"\n"
448 " collection not null, -- name chosen by user\n"
449 " level not null, -- tree level this prefix encodes\n"
450 " prefix not null, -- label identifying node in tree\n"
451 " body not null, -- binary, base64'ed node contents\n"
452 " unique(type, collection, level, prefix)"
453 ");"
454 ;
455
456char const migrate_to_revisions[] =
457 "DROP TABLE schema_version;"
458 "DROP TABLE posting_queue;"
459 "DROP TABLE incoming_queue;"
460 "DROP TABLE sequence_numbers;"
461 "DROP TABLE file_certs;"
462 "DROP TABLE netserver_manifests;"
463 "DROP TABLE merkle_nodes;"
464
465 "CREATE TABLE merkle_nodes"
466 " ( type not null, -- \"key\", \"mcert\", \"fcert\", \"rcert\"\n"
467 " collection not null, -- name chosen by user\n"
468 " level not null, -- tree level this prefix encodes\n"
469 " prefix not null, -- label identifying node in tree\n"
470 " body not null, -- binary, base64'ed node contents\n"
471 " unique(type, collection, level, prefix)"
472 " );"
473
474 "CREATE TABLE revision_certs"
475 " ( hash not null unique, -- hash of remaining fields separated by \":\"\n"
476 " id not null, -- joins with revisions.id\n"
477 " name not null, -- opaque string chosen by user\n"
478 " value not null, -- opaque blob\n"
479 " keypair not null, -- joins with public_keys.id\n"
480 " signature not null, -- RSA/SHA1 signature of \"[name@id:val]\"\n"
481 " unique(name, id, value, keypair, signature)"
482 " );"
483
484 "CREATE TABLE revisions"
485 " ( id primary key, -- SHA1(text of revision)\n"
486 " data not null -- compressed, encoded contents of a revision\n"
487 " );"
488
489 "CREATE TABLE revision_ancestry"
490 " ( parent not null, -- joins with revisions.id\n"
491 " child not null, -- joins with revisions.id\n"
492 " unique(parent, child)"
493 " );"
494 ;
495
496char const migrate_to_epochs[] =
497 "DROP TABLE merkle_nodes;"
498 "CREATE TABLE branch_epochs\n"
499 " ( hash not null unique, -- hash of remaining fields separated by \":\"\n"
500 " branch not null unique, -- joins with revision_certs.value\n"
501 " epoch not null -- random hex-encoded id\n"
502 " );"
503 ;
504
505char const migrate_to_vars[] =
506 "CREATE TABLE db_vars\n"
507 " ( domain not null, -- scope of application of a var\n"
508 " name not null, -- var key\n"
509 " value not null, -- var value\n"
510 " unique(domain, name)"
511 " );"
512 ;
513
514char const migrate_add_indexes[] =
515 "CREATE INDEX revision_ancestry__child ON revision_ancestry (child);"
516 "CREATE INDEX revision_certs__id ON revision_certs (id);"
517 "CREATE INDEX revision_certs__name_value ON revision_certs (name, value);"
518 ;
519
520// There is, perhaps, an argument for turning the logic inside the
521// while-loop into a callback function like unbase64(). However, we'd have
522// to get the key_store in there somehow, and besides I think it's clearer
523// this way.
524static void
525migrate_to_external_privkeys(sqlite3 * db, key_store & keys)
526{
527 {
528 sql stmt(db, 3,
529 "SELECT private_keys.id, private_keys.keydata, public_keys.keydata"
530 " FROM private_keys LEFT OUTER JOIN public_keys"
531 " ON private_keys.id = public_keys.id");
532
533 while (stmt.step())
534 {
535 rsa_keypair_id ident(stmt.column_string(0));
536 base64<old_arc4_rsa_priv_key> old_priv(stmt.column_string(1));
537 base64<rsa_pub_key> pub;
538
539 if (stmt.column_nonnull(2))
540 pub = base64<rsa_pub_key>(stmt.column_string(2));
541
542 P(F("moving key '%s' from database to %s")
543 % ident % keys.get_key_dir());
544 keys.migrate_old_key_pair(ident,
545 decode_base64(old_priv),
546 decode_base64(pub));
547 }
548 }
549
550 sql::exec(db, "DROP TABLE private_keys;");
551}
552
553char const migrate_add_rosters[] =
554 "CREATE TABLE rosters"
555 " ( id primary key, -- strong hash of the roster\n"
556 " data not null -- compressed, encoded contents of the roster\n"
557 " );"
558
559 "CREATE TABLE roster_deltas"
560 " ( id not null, -- strong hash of the roster\n"
561 " base not null, -- joins with either rosters.id or roster_deltas.id\n"
562 " delta not null, -- rdiff to construct current from base\n"
563 " unique(id, base)"
564 " );"
565
566 "CREATE TABLE revision_roster"
567 " ( rev_id primary key, -- joins with revisions.id\n"
568 " roster_id not null -- joins with either rosters.id or roster_deltas.id\n"
569 " );"
570
571 "CREATE TABLE next_roster_node_number"
572 " ( node primary key -- only one entry in this table, ever\n"
573 " );"
574 ;
575
576// I wish I had a form of ALTER TABLE COMMENT on sqlite3
577char const migrate_files_BLOB[] =
578 // change the encoding of file(_delta)s
579 "ALTER TABLE files RENAME TO tmp;"
580 "CREATE TABLE files"
581 " ( id primary key, -- strong hash of file contents\n"
582 " data not null -- compressed contents of a file\n"
583 " );"
584 "INSERT INTO files SELECT id, unbase64(data) FROM tmp;"
585 "DROP TABLE tmp;"
586
587 "ALTER TABLE file_deltas RENAME TO tmp;"
588 "CREATE TABLE file_deltas"
589 " ( id not null, -- strong hash of file contents\n"
590 " base not null, -- joins with files.id or file_deltas.id\n"
591 " delta not null, -- compressed rdiff to construct current from base\n"
592 " unique(id, base)"
593 " );"
594 "INSERT INTO file_deltas SELECT id, base, unbase64(delta) FROM tmp;"
595 "DROP TABLE tmp;"
596
597 // migrate other contents which are accessed by get|put_version
598 "UPDATE manifests SET data=unbase64(data);"
599 "UPDATE manifest_deltas SET delta=unbase64(delta);"
600 "UPDATE rosters SET data=unbase64(data) ;"
601 "UPDATE roster_deltas SET delta=unbase64(delta);"
602 "UPDATE db_vars SET value=unbase64(value), name=unbase64(name);"
603 "UPDATE public_keys SET keydata=unbase64(keydata);"
604 "UPDATE revision_certs SET value=unbase64(value),"
605 " signature=unbase64(signature);"
606 "UPDATE manifest_certs SET value=unbase64(value),"
607 " signature=unbase64(signature);"
608 "UPDATE revisions SET data=unbase64(data);"
609 "UPDATE branch_epochs SET branch=unbase64(branch);"
610 ;
611
612char const migrate_rosters_no_hash[] =
613 "DROP TABLE rosters;"
614 "DROP TABLE roster_deltas;"
615 "DROP TABLE revision_roster;"
616
617 "CREATE TABLE rosters"
618 " ( id primary key, -- a revision id\n"
619 " checksum not null, -- checksum of 'data', to protect against"
620 " disk corruption\n"
621 " data not null -- compressed, encoded contents of the roster\n"
622 " );"
623
624 "CREATE TABLE roster_deltas"
625 " ( id primary key, -- a revision id\n"
626 " checksum not null, -- checksum of 'delta', to protect against"
627 " disk corruption\n"
628 " base not null, -- joins with either rosters.id or roster_deltas.id\n"
629 " delta not null -- rdiff to construct current from base\n"
630 " );"
631 ;
632
633char const migrate_add_heights[] =
634 "CREATE TABLE heights"
635 " ( revision not null,-- joins with revisions.id\n"
636 " height not null,-- complex height, array of big endian u32 integers\n"
637 " unique(revision, height)"
638 " );"
639 ;
640
641// this is a function because it has to refer to the numeric constant
642// defined in schema_migration.hh.
643static void
644migrate_add_ccode(sqlite3 * db, key_store &)
645{
646 string cmd = "PRAGMA user_version = ";
647 cmd += boost::lexical_cast<string>(mtn_creator_code);
648 sql::exec(db, cmd.c_str());
649}
650
651char const migrate_add_heights_index[] =
652 "CREATE INDEX heights__height ON heights (height);"
653 ;
654
655char const migrate_to_binary_hashes[] =
656 "UPDATE files SET id=unhex(id);"
657 "UPDATE file_deltas SET id=unhex(id), base=unhex(base);"
658 "UPDATE revisions SET id=unhex(id);"
659 "UPDATE revision_ancestry SET parent=unhex(parent), child=unhex(child);"
660 "UPDATE heights SET revision=unhex(revision);"
661 "UPDATE rosters SET id=unhex(id), checksum=unhex(checksum);"
662 "UPDATE roster_deltas SET id=unhex(id), base=unhex(base), "
663 " checksum=unhex(checksum);"
664 "UPDATE public_keys SET hash=unhex(hash);"
665
666 // revision_certs also gets a new index, so we recreate the
667 // table completely.
668 "ALTER TABLE revision_certs RENAME TO tmp;\n"
669 "CREATE TABLE revision_certs"
670" ( hash not null unique, -- hash of remaining fields separated by \":\"\n"
671" id not null, -- joins with revisions.id\n"
672" name not null, -- opaque string chosen by user\n"
673" value not null, -- opaque blob\n"
674" keypair not null, -- joins with public_keys.id\n"
675" signature not null, -- RSA/SHA1 signature of \"[name@id:val]\"\n"
676" unique(name, value, id, keypair, signature)\n"
677" );"
678 "INSERT INTO revision_certs SELECT unhex(hash), unhex(id), name, value, keypair, signature FROM tmp;"
679 "DROP TABLE tmp;"
680 "CREATE INDEX revision_certs__id ON revision_certs (id);"
681
682 // We altered a comment on this table, thus we need to recreated it.
683 // Additionally, this is the only schema change, so that we get another
684 // schema hash to upgrade to.
685 "ALTER TABLE branch_epochs RENAME TO tmp;"
686 "CREATE TABLE branch_epochs"
687" ( hash not null unique, -- hash of remaining fields separated by \":\"\n"
688" branch not null unique, -- joins with revision_certs.value\n"
689" epoch not null -- random binary id\n"
690" );"
691 "INSERT INTO branch_epochs SELECT unhex(hash), branch, unhex(epoch) FROM tmp;"
692 "DROP TABLE tmp;"
693
694 // To be able to migrate from pre-roster era, we also need to convert
695 // these deprecated tables
696 "UPDATE manifests SET id=unhex(id);"
697 "UPDATE manifest_deltas SET id=unhex(id), base=unhex(base);"
698 "UPDATE manifest_certs SET id=unhex(id), hash=unhex(hash);"
699 ;
700
701
702// these must be listed in order so that ones listed earlier override ones
703// listed later
704enum upgrade_regime
705 {
706 upgrade_changesetify,
707 upgrade_rosterify,
708 upgrade_regen_caches,
709 upgrade_none,
710 };
711static void
712dump(enum upgrade_regime const & regime, string & out)
713{
714 switch (regime)
715 {
716 case upgrade_changesetify: out = "upgrade_changesetify"; break;
717 case upgrade_rosterify: out = "upgrade_rosterify"; break;
718 case upgrade_regen_caches: out = "upgrade_regen_caches"; break;
719 case upgrade_none: out = "upgrade_none"; break;
720 default: out = (FL("upgrade_regime(%d)") % regime).str(); break;
721 }
722}
723
724typedef void (*migrator_cb)(sqlite3 *, key_store &);
725
726// Exactly one of migrator_sql and migrator_func should be non-null in
727// all entries in migration_events, except the very last.
728struct migration_event
729{
730 char const * id;
731 char const * migrator_sql;
732 migrator_cb migrator_func;
733 upgrade_regime regime;
734};
735
736// IMPORTANT: whenever you modify this to add a new schema version, you must
737// also add a new migration test for the new schema version. See
738// tests/schema_migration for details.
739
740const migration_event migration_events[] = {
741 { "edb5fa6cef65bcb7d0c612023d267c3aeaa1e57a",
742 migrate_merge_url_and_group, 0, upgrade_none },
743
744 { "f042f3c4d0a4f98f6658cbaf603d376acf88ff4b",
745 migrate_add_hashes_and_merkle_trees, 0, upgrade_none },
746
747 { "8929e54f40bf4d3b4aea8b037d2c9263e82abdf4",
748 migrate_to_revisions, 0, upgrade_changesetify },
749
750 { "c1e86588e11ad07fa53e5d294edc043ce1d4005a",
751 migrate_to_epochs, 0, upgrade_none },
752
753 { "40369a7bda66463c5785d160819ab6398b9d44f4",
754 migrate_to_vars, 0, upgrade_none },
755
756 { "e372b508bea9b991816d1c74680f7ae10d2a6d94",
757 migrate_add_indexes, 0, upgrade_none },
758
759 { "1509fd75019aebef5ac3da3a5edf1312393b70e9",
760 0, migrate_to_external_privkeys, upgrade_none },
761
762 { "bd86f9a90b5d552f0be1fa9aee847ea0f317778b",
763 migrate_add_rosters, 0, upgrade_rosterify },
764
765 { "1db80c7cee8fa966913db1a463ed50bf1b0e5b0e",
766 migrate_files_BLOB, 0, upgrade_none },
767
768 { "9d2b5d7b86df00c30ac34fe87a3c20f1195bb2df",
769 migrate_rosters_no_hash, 0, upgrade_regen_caches },
770
771 { "ae196843d368d042f475e3dadfed11e9d7f9f01e",
772 migrate_add_heights, 0, upgrade_regen_caches },
773
774 { "48fd5d84f1e5a949ca093e87e5ac558da6e5956d",
775 0, migrate_add_ccode, upgrade_none },
776
777 { "fe48b0804e0048b87b4cea51b3ab338ba187bdc2",
778 migrate_add_heights_index, 0, upgrade_none },
779
780 { "7ca81b45279403419581d7fde31ed888a80bd34e",
781 migrate_to_binary_hashes, 0, upgrade_none },
782
783 // The last entry in this table should always be the current
784 // schema ID, with 0 for the migrators.
785 { "212dd25a23bfd7bfe030ab910e9d62aa66aa2955", 0, 0, upgrade_none }
786};
787const size_t n_migration_events = (sizeof migration_events
788 / sizeof migration_events[0]);
789
790// unfortunately, this has to be aware of the migration_events array and its
791// limits, lest we crash trying to print the garbage on either side.
792static void
793dump(struct migration_event const * const & mref, string & out)
794{
795 struct migration_event const * m = mref;
796 ptrdiff_t i = m - migration_events;
797 if (m == 0)
798 out = "invalid migration event (null pointer)";
799 else if (i < 0 || static_cast<size_t>(i) >= n_migration_events)
800 out = (FL("invalid migration event, index %ld/%lu")
801 % i % n_migration_events).str();
802 else
803 {
804 char const * type;
805 if (m->migrator_sql)
806 type = "SQL only";
807 else if (m->migrator_func)
808 type = "codeful";
809 else
810 type = "none (current)";
811
812 string regime;
813 dump(m->regime, regime);
814
815 out = (FL("migration %ld/%lu: %s, %s, from %s")
816 % i % n_migration_events % type % regime % m->id).str();
817 }
818}
819
820// The next several functions are concerned with calculating the schema hash
821// and determining whether a database is usable (with or without migration).
822static void
823calculate_schema_id(sqlite3 * db, string & ident)
824{
825 sql stmt(db, 1,
826 "SELECT sql FROM sqlite_master "
827 "WHERE (type = 'table' OR type = 'index') "
828 // filter out NULL statements, because
829 // those are auto-generated indices (for
830 // UNIQUE constraints, etc.).
831 "AND sql IS NOT NULL "
832 "AND name not like 'sqlite_stat%' "
833 "ORDER BY name");
834
835 string schema;
836 using boost::char_separator;
837 typedef boost::tokenizer<char_separator<char> > tokenizer;
838 char_separator<char> sep(" \r\n\t", "(),;");
839
840 while (stmt.step())
841 {
842 string table_schema(stmt.column_string(0));
843 tokenizer tokens(table_schema, sep);
844 for (tokenizer::iterator i = tokens.begin(); i != tokens.end(); i++)
845 {
846 if (schema.size() != 0)
847 schema += " ";
848 schema += *i;
849 }
850 }
851
852 u32 code = sql::value(db, "PRAGMA user_version");
853 if (code != 0)
854 {
855 schema += " PRAGMA user_version = ";
856 schema += boost::lexical_cast<string>(code);
857 }
858
859 id tid;
860 calculate_ident(data(schema), tid);
861 ident = encode_hexenc(tid());
862}
863
864// Look through the migration_events table and return a pointer to the entry
865// corresponding to database DB, or null if it isn't there (i.e. if the
866// database schema is not one we know).
867static migration_event const *
868find_migration(sqlite3 * db)
869{
870 string id;
871 calculate_schema_id(db, id);
872
873 for (migration_event const *m = migration_events + n_migration_events - 1;
874 m >= migration_events; m--)
875 if (m->id == id)
876 return m;
877
878 return 0;
879}
880
881// This enumerates the possible mismatches between the monotone executable
882// and its database.
883enum schema_mismatch_case
884 {
885 SCHEMA_MATCHES = 0,
886 SCHEMA_MIGRATION_NEEDED,
887 SCHEMA_TOO_NEW,
888 SCHEMA_NOT_MONOTONE,
889 SCHEMA_EMPTY
890 };
891static void dump(schema_mismatch_case const & cat, std::string & out)
892{
893 switch (cat)
894 {
895 case SCHEMA_MATCHES: out = "SCHEMA_MATCHES"; break;
896 case SCHEMA_MIGRATION_NEEDED: out = "SCHEMA_MIGRATION_NEEDED"; break;
897 case SCHEMA_TOO_NEW: out = "SCHEMA_TOO_NEW"; break;
898 case SCHEMA_NOT_MONOTONE: out = "SCHEMA_NOT_MONOTONE"; break;
899 case SCHEMA_EMPTY: out = "SCHEMA_EMPTY"; break;
900 default: out = (FL("schema_mismatch_case(%d)") % cat).str(); break;
901 }
902}
903
904
905static schema_mismatch_case
906classify_schema(sqlite3 * db, migration_event const * m = 0)
907{
908 if (!m)
909 m = find_migration(db);
910
911 if (m)
912 {
913 if (m->migrator_sql || m->migrator_func)
914 return SCHEMA_MIGRATION_NEEDED;
915 else
916 return SCHEMA_MATCHES;
917 }
918 else
919 {
920 // Distinguish an utterly empty database, such as is created by
921 // "mtn db load < /dev/null", or by the sqlite3 command line utility
922 // if you don't give it anything to do.
923 if (sql::value(db, "SELECT COUNT(*) FROM sqlite_master") == 0)
924 return SCHEMA_EMPTY;
925
926 // monotone started setting this value in database headers only with
927 // version 0.33, but all previous versions' databases are recognized
928 // by their schema hashes.
929
930 u32 code = sql::value(db, "PRAGMA user_version");
931 if (code != mtn_creator_code)
932 return SCHEMA_NOT_MONOTONE;
933
934 return SCHEMA_TOO_NEW;
935 }
936}
937
938string
939describe_sql_schema(sqlite3 * db)
940{
941 I(db != NULL);
942 string hash;
943 calculate_schema_id(db, hash);
944
945 switch (classify_schema(db))
946 {
947 case SCHEMA_MATCHES:
948 return (F("%s (usable)") % hash).str();
949 case SCHEMA_MIGRATION_NEEDED:
950 return (F("%s (migration needed)") % hash).str();
951 case SCHEMA_TOO_NEW:
952 return (F("%s (too new, cannot use)") % hash).str();
953 case SCHEMA_NOT_MONOTONE:
954 return (F("%s (not a monotone database)") % hash).str();
955 case SCHEMA_EMPTY:
956 return (F("%s (database has no tables!)") % hash).str();
957 default:
958 I(false);
959 }
960}
961
962// Provide sensible diagnostics for a database schema whose hash we do not
963// recognize. (Shared between check_sql_schema and migrate_sql_schema.)
964static void
965diagnose_unrecognized_schema(schema_mismatch_case cat,
966 system_path const & filename)
967{
968 N(cat != SCHEMA_EMPTY,
969 F("cannot use the empty sqlite database %s\n"
970 "(monotone databases must be created with '%s db init')")
971 % filename % ui.prog_name);
972
973 N(cat != SCHEMA_NOT_MONOTONE,
974 F("%s does not appear to be a monotone database\n")
975 % filename);
976
977 N(cat != SCHEMA_TOO_NEW,
978 F("%s appears to be a monotone database, but this version of\n"
979 "monotone does not recognize its schema.\n"
980 "you probably need a newer version of monotone.")
981 % filename);
982}
983
984// check_sql_schema is called by database.cc on open, to determine whether
985// the schema is up to date. If it returns at all, the schema is indeed
986// up to date (otherwise it throws a diagnostic).
987void
988check_sql_schema(sqlite3 * db, system_path const & filename)
989{
990 I(db != NULL);
991
992 schema_mismatch_case cat = classify_schema(db);
993
994 diagnose_unrecognized_schema(cat, filename);
995
996 N(cat != SCHEMA_MIGRATION_NEEDED,
997 F("database %s is laid out according to an old schema\n"
998 "try '%s db migrate' to upgrade\n"
999 "(this is irreversible; you may want to make a backup copy first)")
1000 % filename % ui.prog_name);
1001}
1002
1003void
1004migrate_sql_schema(sqlite3 * db, key_store & keys,
1005 system_path const & filename)
1006{
1007 I(db != NULL);
1008
1009 upgrade_regime regime = upgrade_none; MM(regime);
1010
1011 // Take an exclusive lock on the database before we try to read anything
1012 // from it. If we don't take this lock until the beginning of the
1013 // "migrating data" phase, two simultaneous "db migrate" processes could
1014 // race through the "calculating migration" phase; then one of them would
1015 // wait for the other to finish all the migration steps, and trip over the
1016 // invariant check inside the for loop.
1017 {
1018 transaction guard(db);
1019
1020 P(F("calculating migration..."));
1021
1022 migration_event const *m; MM(m);
1023 schema_mismatch_case cat; MM(cat);
1024 m = find_migration(db);
1025 cat = classify_schema(db, m);
1026
1027 diagnose_unrecognized_schema(cat, filename);
1028
1029 // We really want 'db migrate' on an up-to-date schema to be a no-op
1030 // (no vacuum or anything, even), so that automated scripts can fire
1031 // one off optimistically and not have to worry about getting their
1032 // administrators to do it by hand.
1033 if (cat == SCHEMA_MATCHES)
1034 {
1035 P(F("no migration performed; database schema already up-to-date"));
1036 return;
1037 }
1038
1039 sql::create_function(db, "sha1", sqlite_sha1_fn);
1040 sql::create_function(db, "unbase64", sqlite3_unbase64_fn);
1041 sql::create_function(db, "unhex", sqlite3_unhex_fn);
1042
1043 P(F("migrating data..."));
1044
1045 for (;;)
1046 {
1047 // confirm that we are where we ought to be
1048 string id; MM(id);
1049 calculate_schema_id(db, id);
1050
1051 I(id == m->id);
1052 I(!m->migrator_sql || !m->migrator_func);
1053
1054 if (m->migrator_sql)
1055 sql::exec(db, m->migrator_sql);
1056 else if (m->migrator_func)
1057 m->migrator_func(db, keys);
1058 else
1059 break;
1060
1061 regime = std::min(regime, m->regime);
1062
1063 m++;
1064 I(m < migration_events + n_migration_events);
1065 P(F("migrated to schema %s") % m->id);
1066 }
1067
1068 P(F("committing changes to database"));
1069 guard.commit();
1070 }
1071
1072 P(F("optimizing database"));
1073 sql::exec(db, "VACUUM");
1074
1075 switch (regime)
1076 {
1077 case upgrade_changesetify:
1078 case upgrade_rosterify:
1079 {
1080 string command_str = (regime == upgrade_changesetify
1081 ? "changesetify" : "rosterify");
1082 P(F("NOTE: because this database was last used by a rather old version\n"
1083 "of monotone, you're not done yet. If you're a project leader, then\n"
1084 "see the file UPGRADE for instructions on running '%s db %s'")
1085 % ui.prog_name % command_str);
1086 }
1087 break;
1088 case upgrade_regen_caches:
1089 P(F("NOTE: this upgrade cleared monotone's caches\n"
1090 "you should now run '%s db regenerate_caches'")
1091 % ui.prog_name);
1092 break;
1093 case upgrade_none:
1094 break;
1095 }
1096}
1097
1098// test_migration_step runs the migration step from SCHEMA to its successor,
1099// *without* validating that the database actually conforms to that schema
1100// first. the point of this is to test error recovery from conditions that
1101// are not accessible through normal malformed dumps (because the schema
1102// conformance check will reject them).
1103
1104void
1105test_migration_step(sqlite3 * db, key_store & keys,
1106 system_path const & filename,
1107 string const & schema)
1108{
1109 I(db != NULL);
1110 sql::create_function(db, "sha1", sqlite_sha1_fn);
1111 sql::create_function(db, "unbase64", sqlite3_unbase64_fn);
1112 sql::create_function(db, "unhex", sqlite3_unhex_fn);
1113
1114 transaction guard(db);
1115
1116 migration_event const *m;
1117 for (m = migration_events + n_migration_events - 1;
1118 m >= migration_events; m--)
1119 if (schema == m->id)
1120 break;
1121
1122 N(m >= migration_events,
1123 F("cannot test migration from unknown schema %s") % schema);
1124
1125 N(m->migrator_sql || m->migrator_func,
1126 F("schema %s is up to date") % schema);
1127
1128 L(FL("testing migration from %s to %s\n in database %s")
1129 % schema % m[1].id % filename);
1130
1131 if (m->migrator_sql)
1132 sql::exec(db, m->migrator_sql);
1133 else
1134 m->migrator_func(db, keys);
1135
1136 // in the unlikely event that we get here ...
1137 P(F("successful migration to schema %s") % m[1].id);
1138 guard.commit();
1139}
1140
1141
1142// Local Variables:
1143// mode: C++
1144// fill-column: 76
1145// c-file-style: "gnu"
1146// indent-tabs-mode: nil
1147// End:
1148// vim: et:sw=2:sts=2:ts=2:cino=>2s,{s,\:s,+s,t0,g0,^-2,e-2,n-2,p2s,(0,=s:

Archive Download this file

Branches

Tags

Quick Links:     www.monotone.ca    -     Downloads    -     Documentation    -     Wiki    -     Code Forge    -     Build Status