monotone

monotone Mtn Source Tree

Root/schema_migration.cc

1// Copyright (C) 2002 Graydon Hoare <graydon@pobox.com>
2//
3// This program is made available under the GNU GPL version 2.0 or
4// greater. See the accompanying file COPYING for details.
5//
6// This program is distributed WITHOUT ANY WARRANTY; without even the
7// implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
8// PURPOSE.
9
10#include "base.hh"
11#include <boost/tokenizer.hpp>
12#include "lexical_cast.hh"
13#include "sqlite/sqlite3.h"
14#include <string.h>
15
16#include "sanity.hh"
17#include "schema_migration.hh"
18#include "app_state.hh"
19#include "keys.hh"
20#include "transforms.hh"
21#include "ui.hh"
22
23using std::string;
24
25// this file knows how to migrate schema databases. the general strategy is
26// to hash each schema we ever use, and make a list of the SQL commands
27// required to get from one hash value to the next. when you do a
28// migration, the migrator locates your current db's state on the list and
29// then runs all the migration functions between that point and the target
30// of the migration.
31
32// you will notice a little bit of duplicated code between here and
33// database.cc; this was originally to facilitate inclusion of migration
34// capability into the depot code, but is now preserved because the code
35// in this file is easier to write and understand if it speaks directly
36// to sqlite.
37
38// Wrappers around the bare sqlite3 API. We do not use sqlite3_exec because
39// we want the better error handling that sqlite3_prepare_v2 gives us.
40
41void
42assert_sqlite3_ok(sqlite3 * db)
43{
44 int errcode = sqlite3_errcode(db);
45
46 if (errcode == SQLITE_OK)
47 return;
48
49 char const * errmsg = sqlite3_errmsg(db);
50
51 // first log the code so we can find _out_ what the confusing code
52 // was... note that code does not uniquely identify the errmsg, unlike
53 // errno's.
54 L(FL("sqlite error: %d: %s") % errcode % errmsg);
55
56 // Check the string to see if it looks like an informative_failure
57 // thrown from within an SQL extension function, caught, and turned
58 // into a call to sqlite3_result_error. (Extension functions have to
59 // do this to avoid corrupting sqlite's internal state.) If it is,
60 // rethrow it rather than feeding it to E(), lest we get "error:
61 // sqlite error: error: " ugliness.
62 char const *pfx = _("error: ");
63 if (!std::strncmp(errmsg, pfx, strlen(pfx)))
64 throw informative_failure(errmsg);
65
66 // sometimes sqlite is not very helpful
67 // so we keep a table of errors people have gotten and more helpful versions
68 char const * auxiliary_message = "";
69 switch (errcode)
70 {
71 // All memory-exhaustion conditions should give the same diagnostic.
72 case SQLITE_NOMEM:
73 throw std::bad_alloc();
74
75 // These diagnostics generally indicate an operating-system-level
76 // failure. It would be nice to throw strerror(errno) in there but
77 // we cannot assume errno is still valid by the time we get here.
78 case SQLITE_IOERR:
79 case SQLITE_CANTOPEN:
80 case SQLITE_PROTOCOL:
81 auxiliary_message
82 = _("make sure database and containing directory are writeable\n"
83 "and you have not run out of disk space");
84 break;
85
86 // These error codes may indicate someone is trying to load a database
87 // so old that it's in sqlite 2's disk format (monotone 0.16 or
88 // older).
89 case SQLITE_CORRUPT:
90 case SQLITE_NOTADB:
91 auxiliary_message
92 = _("(if this is a database last used by monotone 0.16 or older,\n"
93 "you must follow a special procedure to make it usable again.\n"
94 "see the file UPGRADE, in the distribution, for instructions.)");
95
96 default:
97 break;
98 }
99
100 // if the auxiliary message is empty, the \n will be stripped off too
101 E(false, F("sqlite error: %s\n%s") % errmsg % auxiliary_message);
102}
103
104
105namespace
106{
107 struct sql
108 {
109 sql(sqlite3 * db, int cols, char const *cmd, char const **afterp = 0)
110 : stmt(0), ncols(cols)
111 {
112 sqlite3_stmt * s;
113
114 char const * after;
115 L(FL("executing SQL '%s'") % cmd);
116
117 sqlite3_prepare_v2(db, cmd, strlen(cmd), &s, &after);
118 assert_sqlite3_ok(db);
119
120 I(s);
121 if (afterp)
122 *afterp = after;
123 else
124 I(*after == 0);
125 I(sqlite3_column_count(s) == ncols);
126 stmt = s;
127 }
128 ~sql()
129 {
130 if (stmt)
131 sqlite3_finalize(stmt);
132 }
133
134 bool step()
135 {
136 int res = sqlite3_step(stmt);
137 if (res == SQLITE_ROW)
138 return true;
139 if (res == SQLITE_DONE)
140 {
141 L(FL("success"));
142 return false;
143 }
144 // Diagnostics from sqlite3_result_error show up in sqlite3_errmsg
145 // only after sqlite3_finalize or sqlite3_reset are called on the
146 // stmt object. See SQLite ticket #1640.
147 sqlite3 * db = sqlite3_db_handle(stmt);
148 sqlite3_finalize(stmt);
149 stmt = 0;
150 assert_sqlite3_ok(db);
151 I(false);
152 }
153 int column_int(int col)
154 {
155 I(col >= 0 && col < ncols);
156 return sqlite3_column_int(stmt, col);
157 }
158 string column_string(int col)
159 {
160 I(col >= 0 && col < ncols);
161 return string(reinterpret_cast<char const *>
162 (sqlite3_column_text(stmt, col)));
163 }
164 bool column_nonnull(int col)
165 {
166 I(col >= 0 && col < ncols);
167 return sqlite3_column_type(stmt, col) != SQLITE_NULL;
168 }
169
170 // convenience for executing a sequence of sql statements,
171 // none of which returns any rows.
172 static void exec(sqlite3 * db, char const * cmd)
173 {
174 do
175 {
176 sql stmt(db, 0, cmd, &cmd);
177 I(stmt.step() == false);
178 }
179 while (*cmd != '\0');
180 }
181
182 // convenience for evaluating an expression that returns a single number.
183 static int value(sqlite3 * db, char const * cmd)
184 {
185 sql stmt(db, 1, cmd);
186
187 I(stmt.step() == true);
188 int res = stmt.column_int(0);
189 I(stmt.step() == false);
190
191 return res;
192 }
193
194 // convenience for making functions
195 static void create_function(sqlite3 * db, char const * name,
196 void (*fn)(sqlite3_context *,
197 int, sqlite3_value **))
198 {
199 sqlite3_create_function(db, name, -1, SQLITE_UTF8, 0, fn, 0, 0);
200 assert_sqlite3_ok(db);
201 }
202
203 private:
204 sqlite3_stmt * stmt;
205 int ncols;
206 };
207
208 struct transaction
209 {
210 transaction(sqlite3 * s) : db(s), committed(false)
211 {
212 sql::exec(db, "BEGIN EXCLUSIVE");
213 }
214 void commit()
215 {
216 I(committed == false);
217 committed = true;
218 }
219 ~transaction()
220 {
221 if (committed)
222 sql::exec(db, "COMMIT");
223 else
224 sql::exec(db, "ROLLBACK");
225 }
226 private:
227 sqlite3 * db;
228 bool committed;
229 };
230}
231
232// SQL extension functions.
233
234// sqlite3_value_text returns unsigned char const *, which is inconvenient
235inline char const *
236sqlite3_value_cstr(sqlite3_value * arg)
237{
238 return reinterpret_cast<char const *>(sqlite3_value_text(arg));
239}
240
241inline bool is_ws(char c)
242{
243 return c == '\r' || c == '\n' || c == '\t' || c == ' ';
244}
245
246static void
247sqlite_sha1_fn(sqlite3_context *f, int nargs, sqlite3_value ** args)
248{
249 if (nargs <= 1)
250 {
251 sqlite3_result_error(f, "need at least 1 arg to sha1()", -1);
252 return;
253 }
254
255 string tmp;
256 if (nargs == 1)
257 {
258 char const * s = sqlite3_value_cstr(args[0]);
259 char const * end = s + sqlite3_value_bytes(args[0]) - 1;
260 remove_copy_if(s, end, back_inserter(tmp), is_ws);
261 }
262 else
263 {
264 char const * sep = sqlite3_value_cstr(args[0]);
265
266 for (int i = 1; i < nargs; ++i)
267 {
268 if (i > 1)
269 tmp += sep;
270 char const * s = sqlite3_value_cstr(args[i]);
271 char const * end = s + sqlite3_value_bytes(args[i]) - 1;
272 remove_copy_if(s, end, back_inserter(tmp), is_ws);
273 }
274 }
275
276 hexenc<id> sha;
277 calculate_ident(data(tmp), sha);
278 sqlite3_result_text(f, sha().c_str(), sha().size(), SQLITE_TRANSIENT);
279}
280
281static void
282sqlite3_unbase64_fn(sqlite3_context *f, int nargs, sqlite3_value ** args)
283{
284 if (nargs != 1)
285 {
286 sqlite3_result_error(f, "need exactly 1 arg to unbase64()", -1);
287 return;
288 }
289 data decoded;
290
291 // This operation may throw informative_failure. We must intercept that
292 // and turn it into a call to sqlite3_result_error, or rollback will fail.
293 try
294 {
295 decode_base64(base64<data>(string(sqlite3_value_cstr(args[0]))), decoded);
296 }
297 catch (informative_failure & e)
298 {
299 sqlite3_result_error(f, e.what(), -1);
300 return;
301 }
302 sqlite3_result_blob(f, decoded().c_str(), decoded().size(), SQLITE_TRANSIENT);
303}
304
305// Here are all of the migration steps. Almost all of them can be expressed
306// entirely as a series of SQL statements; those statements are packaged
307// into a long, continued string constant for the step. One step requires a
308// function instead.
309
310char const migrate_merge_url_and_group[] =
311 // migrate the posting_queue table
312 "ALTER TABLE posting_queue RENAME TO tmp;"
313 "CREATE TABLE posting_queue"
314 " ( url not null, -- URL we are going to send this to\n"
315 " content not null -- the packets we're going to send\n"
316 " );"
317 "INSERT INTO posting_queue"
318 " SELECT (url || '/' || groupname), content FROM tmp;"
319 "DROP TABLE tmp;"
320
321 // migrate the incoming_queue table
322 "ALTER TABLE incoming_queue RENAME TO tmp;"
323 "CREATE TABLE incoming_queue "
324 " ( url not null, -- URL we got this bundle from\n"
325 " content not null -- the packets we're going to read\n"
326 " );"
327 "INSERT INTO incoming_queue"
328 " SELECT (url || '/' || groupname), content FROM tmp;"
329 "DROP TABLE tmp;"
330
331 // migrate the sequence_numbers table
332 "ALTER TABLE sequence_numbers RENAME TO tmp;"
333 "CREATE TABLE sequence_numbers "
334 " ( url primary key, -- URL to read from\n"
335 " major not null, -- 0 in news servers, may be higher in depots\n"
336 " minor not null -- last article / packet sequence number we got\n"
337 " );"
338 "INSERT INTO sequence_numbers"
339 " SELECT (url || '/' || groupname), major, minor FROM tmp;"
340 "DROP TABLE tmp;"
341
342
343 // migrate the netserver_manifests table
344 "ALTER TABLE netserver_manifests RENAME TO tmp;"
345 "CREATE TABLE netserver_manifests"
346 " ( url not null, -- url of some server\n"
347 " manifest not null, -- manifest which exists on url\n"
348 " unique(url, manifest)"
349 " );"
350 "INSERT INTO netserver_manifests"
351 " SELECT (url || '/' || groupname), manifest FROM tmp;"
352
353 "DROP TABLE tmp;"
354 ;
355
356char const migrate_add_hashes_and_merkle_trees[] =
357 // add the column to manifest_certs
358 "ALTER TABLE manifest_certs RENAME TO tmp;"
359 "CREATE TABLE manifest_certs"
360 " ( hash not null unique, -- hash of remaining fields separated by \":\"\n"
361 " id not null, -- joins with manifests.id or manifest_deltas.id\n"
362 " name not null, -- opaque string chosen by user\n"
363 " value not null, -- opaque blob\n"
364 " keypair not null, -- joins with public_keys.id\n"
365 " signature not null, -- RSA/SHA1 signature of \"[name@id:val]\"\n"
366 " unique(name, id, value, keypair, signature)"
367 " );"
368 "INSERT INTO manifest_certs"
369 " SELECT sha1(':', id, name, value, keypair, signature),"
370 " id, name, value, keypair, signature"
371 " FROM tmp;"
372 "DROP TABLE tmp;"
373
374 // add the column to file_certs
375 "ALTER TABLE file_certs RENAME TO tmp;"
376 "CREATE TABLE file_certs"
377 " ( hash not null unique, -- hash of remaining fields separated by \":\"\n"
378 " id not null, -- joins with files.id or file_deltas.id\n"
379 " name not null, -- opaque string chosen by user\n"
380 " value not null, -- opaque blob\n"
381 " keypair not null, -- joins with public_keys.id\n"
382 " signature not null, -- RSA/SHA1 signature of \"[name@id:val]\"\n"
383 " unique(name, id, value, keypair, signature)"
384 " );"
385 "INSERT INTO file_certs"
386 " SELECT sha1(':', id, name, value, keypair, signature),"
387 " id, name, value, keypair, signature"
388 " FROM tmp;"
389 "DROP TABLE tmp;"
390
391 // add the column to public_keys
392 "ALTER TABLE public_keys RENAME TO tmp;"
393 "CREATE TABLE public_keys"
394 " ( hash not null unique, -- hash of remaining fields separated by \":\"\n"
395 " id primary key, -- key identifier chosen by user\n"
396 " keydata not null -- RSA public params\n"
397 " );"
398 "INSERT INTO public_keys SELECT sha1(':',id,keydata), id, keydata FROM tmp;"
399 "DROP TABLE tmp;"
400
401 // add the column to private_keys
402 "ALTER TABLE private_keys RENAME TO tmp;"
403 "CREATE TABLE private_keys"
404 " ( hash not null unique, -- hash of remaining fields separated by \":\"\n"
405 " id primary key, -- as in public_keys (same identifiers, in fact)\n"
406 " keydata not null -- encrypted RSA private params\n"
407 " );"
408 "INSERT INTO private_keys SELECT sha1(':',id,keydata), id, keydata FROM tmp;"
409 "DROP TABLE tmp;"
410
411 // add the merkle tree stuff
412 "CREATE TABLE merkle_nodes"
413 " ( type not null, -- \"key\", \"mcert\", \"fcert\", \"manifest\"\n"
414 " collection not null, -- name chosen by user\n"
415 " level not null, -- tree level this prefix encodes\n"
416 " prefix not null, -- label identifying node in tree\n"
417 " body not null, -- binary, base64'ed node contents\n"
418 " unique(type, collection, level, prefix)"
419 ");"
420 ;
421
422char const migrate_to_revisions[] =
423 "DROP TABLE schema_version;"
424 "DROP TABLE posting_queue;"
425 "DROP TABLE incoming_queue;"
426 "DROP TABLE sequence_numbers;"
427 "DROP TABLE file_certs;"
428 "DROP TABLE netserver_manifests;"
429 "DROP TABLE merkle_nodes;"
430
431 "CREATE TABLE merkle_nodes"
432 " ( type not null, -- \"key\", \"mcert\", \"fcert\", \"rcert\"\n"
433 " collection not null, -- name chosen by user\n"
434 " level not null, -- tree level this prefix encodes\n"
435 " prefix not null, -- label identifying node in tree\n"
436 " body not null, -- binary, base64'ed node contents\n"
437 " unique(type, collection, level, prefix)"
438 " );"
439
440 "CREATE TABLE revision_certs"
441 " ( hash not null unique, -- hash of remaining fields separated by \":\"\n"
442 " id not null, -- joins with revisions.id\n"
443 " name not null, -- opaque string chosen by user\n"
444 " value not null, -- opaque blob\n"
445 " keypair not null, -- joins with public_keys.id\n"
446 " signature not null, -- RSA/SHA1 signature of \"[name@id:val]\"\n"
447 " unique(name, id, value, keypair, signature)"
448 " );"
449
450 "CREATE TABLE revisions"
451 " ( id primary key, -- SHA1(text of revision)\n"
452 " data not null -- compressed, encoded contents of a revision\n"
453 " );"
454
455 "CREATE TABLE revision_ancestry"
456 " ( parent not null, -- joins with revisions.id\n"
457 " child not null, -- joins with revisions.id\n"
458 " unique(parent, child)"
459 " );"
460 ;
461
462char const migrate_to_epochs[] =
463 "DROP TABLE merkle_nodes;"
464 "CREATE TABLE branch_epochs\n"
465 " ( hash not null unique, -- hash of remaining fields separated by \":\"\n"
466 " branch not null unique, -- joins with revision_certs.value\n"
467 " epoch not null -- random hex-encoded id\n"
468 " );"
469 ;
470
471char const migrate_to_vars[] =
472 "CREATE TABLE db_vars\n"
473 " ( domain not null, -- scope of application of a var\n"
474 " name not null, -- var key\n"
475 " value not null, -- var value\n"
476 " unique(domain, name)"
477 " );"
478 ;
479
480char const migrate_add_indexes[] =
481 "CREATE INDEX revision_ancestry__child ON revision_ancestry (child);"
482 "CREATE INDEX revision_certs__id ON revision_certs (id);"
483 "CREATE INDEX revision_certs__name_value ON revision_certs (name, value);"
484 ;
485
486// There is, perhaps, an argument for turning the logic inside the
487// while-loop into a callback function like unbase64(). We would then not
488// need a special case for this step in the master migration loop. However,
489// we'd have to get the app_state in there somehow, we might in the future
490// need to do other things that can't be easily expressed in pure SQL, and
491// besides I think it's clearer this way.
492static void
493migrate_to_external_privkeys(sqlite3 * db, app_state &app)
494{
495 {
496 sql stmt(db, 3,
497 "SELECT private_keys.id, private_keys.keydata, public_keys.keydata"
498 " FROM private_keys LEFT OUTER JOIN public_keys"
499 " ON private_keys.id = public_keys.id");
500
501 while (stmt.step())
502 {
503 rsa_keypair_id ident(stmt.column_string(0));
504 base64< arc4<rsa_priv_key> > old_priv(stmt.column_string(1));
505
506 keypair kp;
507 migrate_private_key(app, ident, old_priv, kp);
508 MM(kp.pub);
509
510 if (stmt.column_nonnull(2))
511 {
512 base64< rsa_pub_key > pub(stmt.column_string(2));
513 MM(pub);
514 N(keys_match(ident, pub, ident, kp.pub),
515 F("public and private keys for %s don't match") % ident);
516 }
517 P(F("moving key '%s' from database to %s")
518 % ident % app.keys.get_key_dir());
519 app.keys.put_key_pair(ident, kp);
520 }
521 }
522
523 sql::exec(db, "DROP TABLE private_keys;");
524}
525
526char const migrate_add_rosters[] =
527 "CREATE TABLE rosters"
528 " ( id primary key, -- strong hash of the roster\n"
529 " data not null -- compressed, encoded contents of the roster\n"
530 " );"
531
532 "CREATE TABLE roster_deltas"
533 " ( id not null, -- strong hash of the roster\n"
534 " base not null, -- joins with either rosters.id or roster_deltas.id\n"
535 " delta not null, -- rdiff to construct current from base\n"
536 " unique(id, base)"
537 " );"
538
539 "CREATE TABLE revision_roster"
540 " ( rev_id primary key, -- joins with revisions.id\n"
541 " roster_id not null -- joins with either rosters.id or roster_deltas.id\n"
542 " );"
543
544 "CREATE TABLE next_roster_node_number"
545 " ( node primary key -- only one entry in this table, ever\n"
546 " );"
547 ;
548
549// I wish I had a form of ALTER TABLE COMMENT on sqlite3
550char const migrate_files_BLOB[] =
551 // change the encoding of file(_delta)s
552 "ALTER TABLE files RENAME TO tmp;"
553 "CREATE TABLE files"
554 " ( id primary key, -- strong hash of file contents\n"
555 " data not null -- compressed contents of a file\n"
556 " );"
557 "INSERT INTO files SELECT id, unbase64(data) FROM tmp;"
558 "DROP TABLE tmp;"
559
560 "ALTER TABLE file_deltas RENAME TO tmp;"
561 "CREATE TABLE file_deltas"
562 " ( id not null, -- strong hash of file contents\n"
563 " base not null, -- joins with files.id or file_deltas.id\n"
564 " delta not null, -- compressed rdiff to construct current from base\n"
565 " unique(id, base)"
566 " );"
567 "INSERT INTO file_deltas SELECT id, base, unbase64(delta) FROM tmp;"
568 "DROP TABLE tmp;"
569
570 // migrate other contents which are accessed by get|put_version
571 "UPDATE manifests SET data=unbase64(data);"
572 "UPDATE manifest_deltas SET delta=unbase64(delta);"
573 "UPDATE rosters SET data=unbase64(data) ;"
574 "UPDATE roster_deltas SET delta=unbase64(delta);"
575 "UPDATE db_vars SET value=unbase64(value), name=unbase64(name);"
576 "UPDATE public_keys SET keydata=unbase64(keydata);"
577 "UPDATE revision_certs SET value=unbase64(value),"
578 " signature=unbase64(signature);"
579 "UPDATE manifest_certs SET value=unbase64(value),"
580 " signature=unbase64(signature);"
581 "UPDATE revisions SET data=unbase64(data);"
582 "UPDATE branch_epochs SET branch=unbase64(branch);"
583 ;
584
585char const migrate_rosters_no_hash[] =
586 "DROP TABLE rosters;"
587 "DROP TABLE roster_deltas;"
588 "DROP TABLE revision_roster;"
589
590 "CREATE TABLE rosters"
591 " ( id primary key, -- a revision id\n"
592 " checksum not null, -- checksum of 'data', to protect against"
593 " disk corruption\n"
594 " data not null -- compressed, encoded contents of the roster\n"
595 " );"
596
597 "CREATE TABLE roster_deltas"
598 " ( id primary key, -- a revision id\n"
599 " checksum not null, -- checksum of 'delta', to protect against"
600 " disk corruption\n"
601 " base not null, -- joins with either rosters.id or roster_deltas.id\n"
602 " delta not null -- rdiff to construct current from base\n"
603 " );"
604 ;
605
606char const migrate_add_heights[] =
607 "CREATE TABLE heights"
608 " ( revision not null,-- joins with revisions.id\n"
609 " height not null,-- complex height, array of big endian u32 integers\n"
610 " unique(revision, height)"
611 " );"
612 ;
613
614char const migrate_add_heights_index[] =
615 "CREATE INDEX heights__height ON heights (height);"
616 ;
617
618// this is a function because it has to refer to the numeric constant
619// defined in schema_migration.hh.
620static void
621migrate_add_ccode(sqlite3 * db, app_state &)
622{
623 string cmd = "PRAGMA user_version = ";
624 cmd += boost::lexical_cast<string>(mtn_creator_code);
625 sql::exec(db, cmd.c_str());
626}
627
628
629// these must be listed in order so that ones listed earlier override ones
630// listed later
631enum upgrade_regime
632 {
633 upgrade_changesetify,
634 upgrade_rosterify,
635 upgrade_regen_caches,
636 upgrade_none,
637 };
638static void
639dump(enum upgrade_regime const & regime, string & out)
640{
641 switch (regime)
642 {
643 case upgrade_changesetify: out = "upgrade_changesetify"; break;
644 case upgrade_rosterify: out = "upgrade_rosterify"; break;
645 case upgrade_regen_caches: out = "upgrade_regen_caches"; break;
646 case upgrade_none: out = "upgrade_none"; break;
647 default: out = (FL("upgrade_regime(%d)") % regime).str(); break;
648 }
649}
650
651typedef void (*migrator_cb)(sqlite3 *, app_state &);
652
653// Exactly one of migrator_sql and migrator_func should be non-null in
654// all entries in migration_events, except the very last.
655struct migration_event
656{
657 char const * id;
658 char const * migrator_sql;
659 migrator_cb migrator_func;
660 upgrade_regime regime;
661};
662
663// IMPORTANT: whenever you modify this to add a new schema version, you must
664// also add a new migration test for the new schema version. See
665// tests/schema_migration for details.
666
667const migration_event migration_events[] = {
668 { "edb5fa6cef65bcb7d0c612023d267c3aeaa1e57a",
669 migrate_merge_url_and_group, 0, upgrade_none },
670
671 { "f042f3c4d0a4f98f6658cbaf603d376acf88ff4b",
672 migrate_add_hashes_and_merkle_trees, 0, upgrade_none },
673
674 { "8929e54f40bf4d3b4aea8b037d2c9263e82abdf4",
675 migrate_to_revisions, 0, upgrade_changesetify },
676
677 { "c1e86588e11ad07fa53e5d294edc043ce1d4005a",
678 migrate_to_epochs, 0, upgrade_none },
679
680 { "40369a7bda66463c5785d160819ab6398b9d44f4",
681 migrate_to_vars, 0, upgrade_none },
682
683 { "e372b508bea9b991816d1c74680f7ae10d2a6d94",
684 migrate_add_indexes, 0, upgrade_none },
685
686 { "1509fd75019aebef5ac3da3a5edf1312393b70e9",
687 0, migrate_to_external_privkeys, upgrade_none },
688
689 { "bd86f9a90b5d552f0be1fa9aee847ea0f317778b",
690 migrate_add_rosters, 0, upgrade_rosterify },
691
692 { "1db80c7cee8fa966913db1a463ed50bf1b0e5b0e",
693 migrate_files_BLOB, 0, upgrade_none },
694
695 { "9d2b5d7b86df00c30ac34fe87a3c20f1195bb2df",
696 migrate_rosters_no_hash, 0, upgrade_regen_caches },
697
698 { "ae196843d368d042f475e3dadfed11e9d7f9f01e",
699 migrate_add_heights, 0, upgrade_regen_caches },
700
701 { "48fd5d84f1e5a949ca093e87e5ac558da6e5956d",
702 0, migrate_add_ccode, upgrade_none },
703
704 { "fe48b0804e0048b87b4cea51b3ab338ba187bdc2",
705 migrate_add_heights_index, 0, upgrade_none },
706
707 // The last entry in this table should always be the current
708 // schema ID, with 0 for the migrators.
709 { "7ca81b45279403419581d7fde31ed888a80bd34e", 0, 0, upgrade_none }
710};
711const size_t n_migration_events = (sizeof migration_events
712 / sizeof migration_events[0]);
713
714// unfortunately, this has to be aware of the migration_events array and its
715// limits, lest we crash trying to print the garbage on either side.
716static void
717dump(struct migration_event const * const & mref, string & out)
718{
719 struct migration_event const * m = mref;
720 ptrdiff_t i = m - migration_events;
721 if (m == 0)
722 out = "invalid migration event (null pointer)";
723 else if (i < 0 || static_cast<size_t>(i) >= n_migration_events)
724 out = (FL("invalid migration event, index %ld/%lu")
725 % i % n_migration_events).str();
726 else
727 {
728 char const * type;
729 if (m->migrator_sql)
730 type = "SQL only";
731 else if (m->migrator_func)
732 type = "codeful";
733 else
734 type = "none (current)";
735
736 string regime;
737 dump(m->regime, regime);
738
739 out = (FL("migration %ld/%lu: %s, %s, from %s")
740 % i % n_migration_events % type % regime % m->id).str();
741 }
742}
743
744// The next several functions are concerned with calculating the schema hash
745// and determining whether a database is usable (with or without migration).
746static void
747calculate_schema_id(sqlite3 * db, string & ident)
748{
749 sql stmt(db, 1,
750 "SELECT sql FROM sqlite_master "
751 "WHERE (type = 'table' OR type = 'index') "
752 // filter out NULL statements, because
753 // those are auto-generated indices (for
754 // UNIQUE constraints, etc.).
755 "AND sql IS NOT NULL "
756 "AND name not like 'sqlite_stat%' "
757 "ORDER BY name");
758
759 string schema;
760 using boost::char_separator;
761 typedef boost::tokenizer<char_separator<char> > tokenizer;
762 char_separator<char> sep(" \r\n\t", "(),;");
763
764 while (stmt.step())
765 {
766 string table_schema(stmt.column_string(0));
767 tokenizer tokens(table_schema, sep);
768 for (tokenizer::iterator i = tokens.begin(); i != tokens.end(); i++)
769 {
770 if (schema.size() != 0)
771 schema += " ";
772 schema += *i;
773 }
774 }
775
776 u32 code = sql::value(db, "PRAGMA user_version");
777 if (code != 0)
778 {
779 schema += " PRAGMA user_version = ";
780 schema += boost::lexical_cast<string>(code);
781 }
782 hexenc<id> tid;
783 calculate_ident(data(schema), tid);
784 ident = tid();
785}
786
787// Look through the migration_events table and return a pointer to the entry
788// corresponding to database DB, or null if it isn't there (i.e. if the
789// database schema is not one we know).
790static migration_event const *
791find_migration(sqlite3 * db)
792{
793 string id;
794 calculate_schema_id(db, id);
795
796 for (migration_event const *m = migration_events + n_migration_events - 1;
797 m >= migration_events; m--)
798 if (m->id == id)
799 return m;
800
801 return 0;
802}
803
804// This enumerates the possible mismatches between the monotone executable
805// and its database.
806enum schema_mismatch_case
807 {
808 SCHEMA_MATCHES = 0,
809 SCHEMA_MIGRATION_NEEDED,
810 SCHEMA_TOO_NEW,
811 SCHEMA_NOT_MONOTONE,
812 SCHEMA_EMPTY
813 };
814static void dump(schema_mismatch_case const & cat, std::string & out)
815{
816 switch (cat)
817 {
818 case SCHEMA_MATCHES: out = "SCHEMA_MATCHES"; break;
819 case SCHEMA_MIGRATION_NEEDED: out = "SCHEMA_MIGRATION_NEEDED"; break;
820 case SCHEMA_TOO_NEW: out = "SCHEMA_TOO_NEW"; break;
821 case SCHEMA_NOT_MONOTONE: out = "SCHEMA_NOT_MONOTONE"; break;
822 case SCHEMA_EMPTY: out = "SCHEMA_EMPTY"; break;
823 default: out = (FL("schema_mismatch_case(%d)") % cat).str(); break;
824 }
825}
826
827
828static schema_mismatch_case
829classify_schema(sqlite3 * db, migration_event const * m = 0)
830{
831 if (!m)
832 m = find_migration(db);
833
834 if (m)
835 {
836 if (m->migrator_sql || m->migrator_func)
837 return SCHEMA_MIGRATION_NEEDED;
838 else
839 return SCHEMA_MATCHES;
840 }
841 else
842 {
843 // Distinguish an utterly empty database, such as is created by
844 // "mtn db load < /dev/null", or by the sqlite3 command line utility
845 // if you don't give it anything to do.
846 if (sql::value(db, "SELECT COUNT(*) FROM sqlite_master") == 0)
847 return SCHEMA_EMPTY;
848
849 // monotone started setting this value in database headers only with
850 // version 0.33, but all previous versions' databases are recognized
851 // by their schema hashes.
852
853 u32 code = sql::value(db, "PRAGMA user_version");
854 if (code != mtn_creator_code)
855 return SCHEMA_NOT_MONOTONE;
856
857 return SCHEMA_TOO_NEW;
858 }
859}
860
861string
862describe_sql_schema(sqlite3 * db)
863{
864 I(db != NULL);
865 string hash;
866 calculate_schema_id(db, hash);
867
868 switch (classify_schema(db))
869 {
870 case SCHEMA_MATCHES:
871 return (F("%s (usable)") % hash).str();
872 case SCHEMA_MIGRATION_NEEDED:
873 return (F("%s (migration needed)") % hash).str();
874 case SCHEMA_TOO_NEW:
875 return (F("%s (too new, cannot use)") % hash).str();
876 case SCHEMA_NOT_MONOTONE:
877 return (F("%s (not a monotone database)") % hash).str();
878 case SCHEMA_EMPTY:
879 return (F("%s (database has no tables!)") % hash).str();
880 default:
881 I(false);
882 }
883}
884
885// Provide sensible diagnostics for a database schema whose hash we do not
886// recognize. (Shared between check_sql_schema and migrate_sql_schema.)
887static void
888diagnose_unrecognized_schema(schema_mismatch_case cat,
889 system_path const & filename)
890{
891 N(cat != SCHEMA_EMPTY,
892 F("cannot use the empty sqlite database %s\n"
893 "(monotone databases must be created with '%s db init')")
894 % filename % ui.prog_name);
895
896 N(cat != SCHEMA_NOT_MONOTONE,
897 F("%s does not appear to be a monotone database\n")
898 % filename);
899
900 N(cat != SCHEMA_TOO_NEW,
901 F("%s appears to be a monotone database, but this version of\n"
902 "monotone does not recognize its schema.\n"
903 "you probably need a newer version of monotone.")
904 % filename);
905}
906
907// check_sql_schema is called by database.cc on open, to determine whether
908// the schema is up to date. If it returns at all, the schema is indeed
909// up to date (otherwise it throws a diagnostic).
910void
911check_sql_schema(sqlite3 * db, system_path const & filename)
912{
913 I(db != NULL);
914
915 schema_mismatch_case cat = classify_schema(db);
916
917 diagnose_unrecognized_schema(cat, filename);
918
919 N(cat != SCHEMA_MIGRATION_NEEDED,
920 F("database %s is laid out according to an old schema\n"
921 "try '%s db migrate' to upgrade\n"
922 "(this is irreversible; you may want to make a backup copy first)")
923 % filename % ui.prog_name);
924}
925
926void
927migrate_sql_schema(sqlite3 * db, app_state & app)
928{
929 I(db != NULL);
930
931 upgrade_regime regime = upgrade_none; MM(regime);
932
933 // Take an exclusive lock on the database before we try to read anything
934 // from it. If we don't take this lock until the beginning of the
935 // "migrating data" phase, two simultaneous "db migrate" processes could
936 // race through the "calculating migration" phase; then one of them would
937 // wait for the other to finish all the migration steps, and trip over the
938 // invariant check inside the for loop.
939 {
940 transaction guard(db);
941
942 P(F("calculating migration..."));
943
944 migration_event const *m; MM(m);
945 schema_mismatch_case cat; MM(cat);
946 m = find_migration(db);
947 cat = classify_schema(db, m);
948
949 diagnose_unrecognized_schema(cat, app.db.get_filename());
950
951 // We really want 'db migrate' on an up-to-date schema to be a no-op
952 // (no vacuum or anything, even), so that automated scripts can fire
953 // one off optimistically and not have to worry about getting their
954 // administrators to do it by hand.
955 if (cat == SCHEMA_MATCHES)
956 {
957 P(F("no migration performed; database schema already up-to-date"));
958 return;
959 }
960
961 sql::create_function(db, "sha1", sqlite_sha1_fn);
962 sql::create_function(db, "unbase64", sqlite3_unbase64_fn);
963
964 P(F("migrating data..."));
965
966 for (;;)
967 {
968 // confirm that we are where we ought to be
969 string id; MM(id);
970 calculate_schema_id(db, id);
971
972 I(id == m->id);
973 I(!m->migrator_sql || !m->migrator_func);
974
975 if (m->migrator_sql)
976 sql::exec(db, m->migrator_sql);
977 else if (m->migrator_func)
978 m->migrator_func(db, app);
979 else
980 break;
981
982 regime = std::min(regime, m->regime);
983
984 m++;
985 I(m < migration_events + n_migration_events);
986 P(F("migrated to schema %s") % m->id);
987 }
988
989 P(F("committing changes to database"));
990 guard.commit();
991 }
992
993 P(F("optimizing database"));
994 sql::exec(db, "VACUUM");
995
996 switch (regime)
997 {
998 case upgrade_changesetify:
999 case upgrade_rosterify:
1000 {
1001 string command_str = (regime == upgrade_changesetify
1002 ? "changesetify" : "rosterify");
1003 P(F("NOTE: because this database was last used by a rather old version\n"
1004 "of monotone, you're not done yet. If you're a project leader, then\n"
1005 "see the file UPGRADE for instructions on running '%s db %s'")
1006 % ui.prog_name % command_str);
1007 }
1008 break;
1009 case upgrade_regen_caches:
1010 P(F("NOTE: this upgrade cleared monotone's caches\n"
1011 "you should now run '%s db regenerate_caches'")
1012 % ui.prog_name);
1013 break;
1014 case upgrade_none:
1015 break;
1016 }
1017}
1018
1019// test_migration_step runs the migration step from SCHEMA to its successor,
1020// *without* validating that the database actually conforms to that schema
1021// first. the point of this is to test error recovery from conditions that
1022// are not accessible through normal malformed dumps (because the schema
1023// conformance check will reject them).
1024
1025void
1026test_migration_step(sqlite3 * db, app_state & app, string const & schema)
1027{
1028 I(db != NULL);
1029 sql::create_function(db, "sha1", sqlite_sha1_fn);
1030 sql::create_function(db, "unbase64", sqlite3_unbase64_fn);
1031
1032 transaction guard(db);
1033
1034 migration_event const *m;
1035 for (m = migration_events + n_migration_events - 1;
1036 m >= migration_events; m--)
1037 if (schema == m->id)
1038 break;
1039
1040 N(m >= migration_events,
1041 F("cannot test migration from unknown schema %s") % schema);
1042
1043 N(m->migrator_sql || m->migrator_func,
1044 F("schema %s is up to date") % schema);
1045
1046 L(FL("testing migration from %s to %s\n in database %s")
1047 % schema % m[1].id % app.db.get_filename());
1048
1049 if (m->migrator_sql)
1050 sql::exec(db, m->migrator_sql);
1051 else
1052 m->migrator_func(db, app);
1053
1054 // in the unlikely event that we get here ...
1055 P(F("successful migration to schema %s") % m[1].id);
1056 guard.commit();
1057}
1058
1059
1060// Local Variables:
1061// mode: C++
1062// fill-column: 76
1063// c-file-style: "gnu"
1064// indent-tabs-mode: nil
1065// End:
1066// vim: et:sw=2:sts=2:ts=2:cino=>2s,{s,\:s,+s,t0,g0,^-2,e-2,n-2,p2s,(0,=s:

Archive Download this file

Branches

Tags

Quick Links:     www.monotone.ca    -     Downloads    -     Documentation    -     Wiki    -     Code Forge    -     Build Status