monotone

monotone Mtn Source Tree

Root/schema_migration.cc

1// Copyright (C) 2002 Graydon Hoare <graydon@pobox.com>
2//
3// This program is made available under the GNU GPL version 2.0 or
4// greater. See the accompanying file COPYING for details.
5//
6// This program is distributed WITHOUT ANY WARRANTY; without even the
7// implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
8// PURPOSE.
9
10#include <boost/tokenizer.hpp>
11#include <boost/lexical_cast.hpp>
12#include <sqlite3.h>
13#include <string.h>
14
15#include "sanity.hh"
16#include "schema_migration.hh"
17#include "app_state.hh"
18#include "keys.hh"
19#include "transforms.hh"
20#include "ui.hh"
21
22using std::string;
23
24// this file knows how to migrate schema databases. the general strategy is
25// to hash each schema we ever use, and make a list of the SQL commands
26// required to get from one hash value to the next. when you do a
27// migration, the migrator locates your current db's state on the list and
28// then runs all the migration functions between that point and the target
29// of the migration.
30
31// you will notice a little bit of duplicated code between here and
32// database.cc; this was originally to facilitate inclusion of migration
33// capability into the depot code, but is now preserved because the code
34// in this file is easier to write and understand if it speaks directly
35// to sqlite.
36
37// Wrappers around the bare sqlite3 API. We do not use sqlite3_exec because
38// we want the better error handling that sqlite3_prepare_v2 gives us.
39
40void
41assert_sqlite3_ok(sqlite3 * db)
42{
43 int errcode = sqlite3_errcode(db);
44
45 if (errcode == SQLITE_OK)
46 return;
47
48 char const * errmsg = sqlite3_errmsg(db);
49
50 // first log the code so we can find _out_ what the confusing code
51 // was... note that code does not uniquely identify the errmsg, unlike
52 // errno's.
53 L(FL("sqlite error: %d: %s") % errcode % errmsg);
54
55 // Check the string to see if it looks like an informative_failure
56 // thrown from within an SQL extension function, caught, and turned
57 // into a call to sqlite3_result_error. (Extension functions have to
58 // do this to avoid corrupting sqlite's internal state.) If it is,
59 // rethrow it rather than feeding it to E(), lest we get "error:
60 // sqlite error: error: " ugliness.
61 char const *pfx = _("error: ");
62 if (!std::strncmp(errmsg, pfx, strlen(pfx)))
63 throw informative_failure(errmsg);
64
65 // sometimes sqlite is not very helpful
66 // so we keep a table of errors people have gotten and more helpful versions
67 char const * auxiliary_message = "";
68 switch (errcode)
69 {
70 // All memory-exhaustion conditions should give the same diagnostic.
71 case SQLITE_NOMEM:
72 throw std::bad_alloc();
73
74 // These diagnostics generally indicate an operating-system-level
75 // failure. It would be nice to throw strerror(errno) in there but
76 // we cannot assume errno is still valid by the time we get here.
77 case SQLITE_IOERR:
78 case SQLITE_CANTOPEN:
79 case SQLITE_PROTOCOL:
80 auxiliary_message
81 = _("make sure database and containing directory are writeable\n"
82 "and you have not run out of disk space");
83 break;
84
85 // These error codes may indicate someone is trying to load a database
86 // so old that it's in sqlite 2's disk format (monotone 0.16 or
87 // older).
88 case SQLITE_CORRUPT:
89 case SQLITE_NOTADB:
90 auxiliary_message
91 = _("(if this is a database last used by monotone 0.16 or older,\n"
92 "you must follow a special procedure to make it usable again.\n"
93 "see the file UPGRADE, in the distribution, for instructions.)");
94
95 default:
96 break;
97 }
98
99 // if the auxiliary message is empty, the \n will be stripped off too
100 E(false, F("sqlite error: %s\n%s") % errmsg % auxiliary_message);
101}
102
103
104namespace
105{
106 struct sql
107 {
108 sql(sqlite3 * db, int cols, char const *cmd, char const **afterp = 0)
109 : stmt(0), ncols(cols)
110 {
111 sqlite3_stmt * s;
112
113 char const * after;
114 L(FL("executing SQL '%s'") % cmd);
115
116 sqlite3_prepare_v2(db, cmd, strlen(cmd), &s, &after);
117 assert_sqlite3_ok(db);
118
119 I(s);
120 if (afterp)
121 *afterp = after;
122 else
123 I(*after == 0);
124 I(sqlite3_column_count(s) == ncols);
125 stmt = s;
126 }
127 ~sql()
128 {
129 if (stmt)
130 sqlite3_finalize(stmt);
131 }
132
133 bool step()
134 {
135 int res = sqlite3_step(stmt);
136 if (res == SQLITE_ROW)
137 return true;
138 if (res == SQLITE_DONE)
139 {
140 L(FL("success"));
141 return false;
142 }
143 // Diagnostics from sqlite3_result_error show up in sqlite3_errmsg
144 // only after sqlite3_finalize or sqlite3_reset are called on the
145 // stmt object. See SQLite ticket #1640.
146 sqlite3 * db = sqlite3_db_handle(stmt);
147 sqlite3_finalize(stmt);
148 stmt = 0;
149 assert_sqlite3_ok(db);
150 I(false);
151 }
152 int column_int(int col)
153 {
154 I(col >= 0 && col < ncols);
155 return sqlite3_column_int(stmt, col);
156 }
157 string column_string(int col)
158 {
159 I(col >= 0 && col < ncols);
160 return string(reinterpret_cast<char const *>
161 (sqlite3_column_text(stmt, col)));
162 }
163 bool column_nonnull(int col)
164 {
165 I(col >= 0 && col < ncols);
166 return sqlite3_column_type(stmt, col) != SQLITE_NULL;
167 }
168
169 // convenience for executing a sequence of sql statements,
170 // none of which returns any rows.
171 static void exec(sqlite3 * db, char const * cmd)
172 {
173 do
174 {
175 sql stmt(db, 0, cmd, &cmd);
176 I(stmt.step() == false);
177 }
178 while (*cmd != '\0');
179 }
180
181 // convenience for evaluating an expression that returns a single number.
182 static int value(sqlite3 * db, char const * cmd)
183 {
184 sql stmt(db, 1, cmd);
185
186 I(stmt.step() == true);
187 int res = stmt.column_int(0);
188 I(stmt.step() == false);
189
190 return res;
191 }
192
193 // convenience for making functions
194 static void create_function(sqlite3 * db, char const * name,
195 void (*fn)(sqlite3_context *,
196 int, sqlite3_value **))
197 {
198 sqlite3_create_function(db, name, -1, SQLITE_UTF8, 0, fn, 0, 0);
199 assert_sqlite3_ok(db);
200 }
201
202 private:
203 sqlite3_stmt * stmt;
204 int ncols;
205 };
206
207 struct transaction
208 {
209 transaction(sqlite3 * s) : db(s), committed(false)
210 {
211 sql::exec(db, "BEGIN EXCLUSIVE");
212 }
213 void commit()
214 {
215 I(committed == false);
216 committed = true;
217 }
218 ~transaction()
219 {
220 if (committed)
221 sql::exec(db, "COMMIT");
222 else
223 sql::exec(db, "ROLLBACK");
224 }
225 private:
226 sqlite3 * db;
227 bool committed;
228 };
229}
230
231// SQL extension functions.
232
233// sqlite3_value_text returns unsigned char const *, which is inconvenient
234inline char const *
235sqlite3_value_cstr(sqlite3_value * arg)
236{
237 return reinterpret_cast<char const *>(sqlite3_value_text(arg));
238}
239
240inline bool is_ws(char c)
241{
242 return c == '\r' || c == '\n' || c == '\t' || c == ' ';
243}
244
245static void
246sqlite_sha1_fn(sqlite3_context *f, int nargs, sqlite3_value ** args)
247{
248 if (nargs <= 1)
249 {
250 sqlite3_result_error(f, "need at least 1 arg to sha1()", -1);
251 return;
252 }
253
254 string tmp;
255 if (nargs == 1)
256 {
257 char const * s = sqlite3_value_cstr(args[0]);
258 char const * end = s + sqlite3_value_bytes(args[0]) - 1;
259 remove_copy_if(s, end, back_inserter(tmp), is_ws);
260 }
261 else
262 {
263 char const * sep = sqlite3_value_cstr(args[0]);
264
265 for (int i = 1; i < nargs; ++i)
266 {
267 if (i > 1)
268 tmp += sep;
269 char const * s = sqlite3_value_cstr(args[i]);
270 char const * end = s + sqlite3_value_bytes(args[i]) - 1;
271 remove_copy_if(s, end, back_inserter(tmp), is_ws);
272 }
273 }
274
275 hexenc<id> sha;
276 calculate_ident(data(tmp), sha);
277 sqlite3_result_text(f, sha().c_str(), sha().size(), SQLITE_TRANSIENT);
278}
279
280static void
281sqlite3_unbase64_fn(sqlite3_context *f, int nargs, sqlite3_value ** args)
282{
283 if (nargs != 1)
284 {
285 sqlite3_result_error(f, "need exactly 1 arg to unbase64()", -1);
286 return;
287 }
288 data decoded;
289
290 // This operation may throw informative_failure. We must intercept that
291 // and turn it into a call to sqlite3_result_error, or rollback will fail.
292 try
293 {
294 decode_base64(base64<data>(string(sqlite3_value_cstr(args[0]))), decoded);
295 }
296 catch (informative_failure & e)
297 {
298 sqlite3_result_error(f, e.what(), -1);
299 return;
300 }
301 sqlite3_result_blob(f, decoded().c_str(), decoded().size(), SQLITE_TRANSIENT);
302}
303
304// Here are all of the migration steps. Almost all of them can be expressed
305// entirely as a series of SQL statements; those statements are packaged
306// into a long, continued string constant for the step. One step requires a
307// function instead.
308
309char const migrate_merge_url_and_group[] =
310 // migrate the posting_queue table
311 "ALTER TABLE posting_queue RENAME TO tmp;"
312 "CREATE TABLE posting_queue"
313 " ( url not null, -- URL we are going to send this to\n"
314 " content not null -- the packets we're going to send\n"
315 " );"
316 "INSERT INTO posting_queue"
317 " SELECT (url || '/' || groupname), content FROM tmp;"
318 "DROP TABLE tmp;"
319
320 // migrate the incoming_queue table
321 "ALTER TABLE incoming_queue RENAME TO tmp;"
322 "CREATE TABLE incoming_queue "
323 " ( url not null, -- URL we got this bundle from\n"
324 " content not null -- the packets we're going to read\n"
325 " );"
326 "INSERT INTO incoming_queue"
327 " SELECT (url || '/' || groupname), content FROM tmp;"
328 "DROP TABLE tmp;"
329
330 // migrate the sequence_numbers table
331 "ALTER TABLE sequence_numbers RENAME TO tmp;"
332 "CREATE TABLE sequence_numbers "
333 " ( url primary key, -- URL to read from\n"
334 " major not null, -- 0 in news servers, may be higher in depots\n"
335 " minor not null -- last article / packet sequence number we got\n"
336 " );"
337 "INSERT INTO sequence_numbers"
338 " SELECT (url || '/' || groupname), major, minor FROM tmp;"
339 "DROP TABLE tmp;"
340
341
342 // migrate the netserver_manifests table
343 "ALTER TABLE netserver_manifests RENAME TO tmp;"
344 "CREATE TABLE netserver_manifests"
345 " ( url not null, -- url of some server\n"
346 " manifest not null, -- manifest which exists on url\n"
347 " unique(url, manifest)"
348 " );"
349 "INSERT INTO netserver_manifests"
350 " SELECT (url || '/' || groupname), manifest FROM tmp;"
351
352 "DROP TABLE tmp;"
353 ;
354
355char const migrate_add_hashes_and_merkle_trees[] =
356 // add the column to manifest_certs
357 "ALTER TABLE manifest_certs RENAME TO tmp;"
358 "CREATE TABLE manifest_certs"
359 " ( hash not null unique, -- hash of remaining fields separated by \":\"\n"
360 " id not null, -- joins with manifests.id or manifest_deltas.id\n"
361 " name not null, -- opaque string chosen by user\n"
362 " value not null, -- opaque blob\n"
363 " keypair not null, -- joins with public_keys.id\n"
364 " signature not null, -- RSA/SHA1 signature of \"[name@id:val]\"\n"
365 " unique(name, id, value, keypair, signature)"
366 " );"
367 "INSERT INTO manifest_certs"
368 " SELECT sha1(':', id, name, value, keypair, signature),"
369 " id, name, value, keypair, signature"
370 " FROM tmp;"
371 "DROP TABLE tmp;"
372
373 // add the column to file_certs
374 "ALTER TABLE file_certs RENAME TO tmp;"
375 "CREATE TABLE file_certs"
376 " ( hash not null unique, -- hash of remaining fields separated by \":\"\n"
377 " id not null, -- joins with files.id or file_deltas.id\n"
378 " name not null, -- opaque string chosen by user\n"
379 " value not null, -- opaque blob\n"
380 " keypair not null, -- joins with public_keys.id\n"
381 " signature not null, -- RSA/SHA1 signature of \"[name@id:val]\"\n"
382 " unique(name, id, value, keypair, signature)"
383 " );"
384 "INSERT INTO file_certs"
385 " SELECT sha1(':', id, name, value, keypair, signature),"
386 " id, name, value, keypair, signature"
387 " FROM tmp;"
388 "DROP TABLE tmp;"
389
390 // add the column to public_keys
391 "ALTER TABLE public_keys RENAME TO tmp;"
392 "CREATE TABLE public_keys"
393 " ( hash not null unique, -- hash of remaining fields separated by \":\"\n"
394 " id primary key, -- key identifier chosen by user\n"
395 " keydata not null -- RSA public params\n"
396 " );"
397 "INSERT INTO public_keys SELECT sha1(':',id,keydata), id, keydata FROM tmp;"
398 "DROP TABLE tmp;"
399
400 // add the column to private_keys
401 "ALTER TABLE private_keys RENAME TO tmp;"
402 "CREATE TABLE private_keys"
403 " ( hash not null unique, -- hash of remaining fields separated by \":\"\n"
404 " id primary key, -- as in public_keys (same identifiers, in fact)\n"
405 " keydata not null -- encrypted RSA private params\n"
406 " );"
407 "INSERT INTO private_keys SELECT sha1(':',id,keydata), id, keydata FROM tmp;"
408 "DROP TABLE tmp;"
409
410 // add the merkle tree stuff
411 "CREATE TABLE merkle_nodes"
412 " ( type not null, -- \"key\", \"mcert\", \"fcert\", \"manifest\"\n"
413 " collection not null, -- name chosen by user\n"
414 " level not null, -- tree level this prefix encodes\n"
415 " prefix not null, -- label identifying node in tree\n"
416 " body not null, -- binary, base64'ed node contents\n"
417 " unique(type, collection, level, prefix)"
418 ");"
419 ;
420
421char const migrate_to_revisions[] =
422 "DROP TABLE schema_version;"
423 "DROP TABLE posting_queue;"
424 "DROP TABLE incoming_queue;"
425 "DROP TABLE sequence_numbers;"
426 "DROP TABLE file_certs;"
427 "DROP TABLE netserver_manifests;"
428 "DROP TABLE merkle_nodes;"
429
430 "CREATE TABLE merkle_nodes"
431 " ( type not null, -- \"key\", \"mcert\", \"fcert\", \"rcert\"\n"
432 " collection not null, -- name chosen by user\n"
433 " level not null, -- tree level this prefix encodes\n"
434 " prefix not null, -- label identifying node in tree\n"
435 " body not null, -- binary, base64'ed node contents\n"
436 " unique(type, collection, level, prefix)"
437 " );"
438
439 "CREATE TABLE revision_certs"
440 " ( hash not null unique, -- hash of remaining fields separated by \":\"\n"
441 " id not null, -- joins with revisions.id\n"
442 " name not null, -- opaque string chosen by user\n"
443 " value not null, -- opaque blob\n"
444 " keypair not null, -- joins with public_keys.id\n"
445 " signature not null, -- RSA/SHA1 signature of \"[name@id:val]\"\n"
446 " unique(name, id, value, keypair, signature)"
447 " );"
448
449 "CREATE TABLE revisions"
450 " ( id primary key, -- SHA1(text of revision)\n"
451 " data not null -- compressed, encoded contents of a revision\n"
452 " );"
453
454 "CREATE TABLE revision_ancestry"
455 " ( parent not null, -- joins with revisions.id\n"
456 " child not null, -- joins with revisions.id\n"
457 " unique(parent, child)"
458 " );"
459 ;
460
461char const migrate_to_epochs[] =
462 "DROP TABLE merkle_nodes;"
463 "CREATE TABLE branch_epochs\n"
464 " ( hash not null unique, -- hash of remaining fields separated by \":\"\n"
465 " branch not null unique, -- joins with revision_certs.value\n"
466 " epoch not null -- random hex-encoded id\n"
467 " );"
468 ;
469
470char const migrate_to_vars[] =
471 "CREATE TABLE db_vars\n"
472 " ( domain not null, -- scope of application of a var\n"
473 " name not null, -- var key\n"
474 " value not null, -- var value\n"
475 " unique(domain, name)"
476 " );"
477 ;
478
479char const migrate_add_indexes[] =
480 "CREATE INDEX revision_ancestry__child ON revision_ancestry (child);"
481 "CREATE INDEX revision_certs__id ON revision_certs (id);"
482 "CREATE INDEX revision_certs__name_value ON revision_certs (name, value);"
483 ;
484
485// There is, perhaps, an argument for turning the logic inside the
486// while-loop into a callback function like unbase64(). We would then not
487// need a special case for this step in the master migration loop. However,
488// we'd have to get the app_state in there somehow, we might in the future
489// need to do other things that can't be easily expressed in pure SQL, and
490// besides I think it's clearer this way.
491static void
492migrate_to_external_privkeys(sqlite3 * db, app_state &app)
493{
494 {
495 sql stmt(db, 3,
496 "SELECT private_keys.id, private_keys.keydata, public_keys.keydata"
497 " FROM private_keys LEFT OUTER JOIN public_keys"
498 " ON private_keys.id = public_keys.id");
499
500 while (stmt.step())
501 {
502 rsa_keypair_id ident(stmt.column_string(0));
503 base64< arc4<rsa_priv_key> > old_priv(stmt.column_string(1));
504
505 keypair kp;
506 migrate_private_key(app, ident, old_priv, kp);
507 MM(kp.pub);
508
509 if (stmt.column_nonnull(2))
510 {
511 base64< rsa_pub_key > pub(stmt.column_string(2));
512 MM(pub);
513 N(keys_match(ident, pub, ident, kp.pub),
514 F("public and private keys for %s don't match") % ident);
515 }
516 P(F("moving key '%s' from database to %s")
517 % ident % app.keys.get_key_dir());
518 app.keys.put_key_pair(ident, kp);
519 }
520 }
521
522 sql::exec(db, "DROP TABLE private_keys;");
523}
524
525char const migrate_add_rosters[] =
526 "CREATE TABLE rosters"
527 " ( id primary key, -- strong hash of the roster\n"
528 " data not null -- compressed, encoded contents of the roster\n"
529 " );"
530
531 "CREATE TABLE roster_deltas"
532 " ( id not null, -- strong hash of the roster\n"
533 " base not null, -- joins with either rosters.id or roster_deltas.id\n"
534 " delta not null, -- rdiff to construct current from base\n"
535 " unique(id, base)"
536 " );"
537
538 "CREATE TABLE revision_roster"
539 " ( rev_id primary key, -- joins with revisions.id\n"
540 " roster_id not null -- joins with either rosters.id or roster_deltas.id\n"
541 " );"
542
543 "CREATE TABLE next_roster_node_number"
544 " ( node primary key -- only one entry in this table, ever\n"
545 " );"
546 ;
547
548// I wish I had a form of ALTER TABLE COMMENT on sqlite3
549char const migrate_files_BLOB[] =
550 // change the encoding of file(_delta)s
551 "ALTER TABLE files RENAME TO tmp;"
552 "CREATE TABLE files"
553 " ( id primary key, -- strong hash of file contents\n"
554 " data not null -- compressed contents of a file\n"
555 " );"
556 "INSERT INTO files SELECT id, unbase64(data) FROM tmp;"
557 "DROP TABLE tmp;"
558
559 "ALTER TABLE file_deltas RENAME TO tmp;"
560 "CREATE TABLE file_deltas"
561 " ( id not null, -- strong hash of file contents\n"
562 " base not null, -- joins with files.id or file_deltas.id\n"
563 " delta not null, -- compressed rdiff to construct current from base\n"
564 " unique(id, base)"
565 " );"
566 "INSERT INTO file_deltas SELECT id, base, unbase64(delta) FROM tmp;"
567 "DROP TABLE tmp;"
568
569 // migrate other contents which are accessed by get|put_version
570 "UPDATE manifests SET data=unbase64(data);"
571 "UPDATE manifest_deltas SET delta=unbase64(delta);"
572 "UPDATE rosters SET data=unbase64(data) ;"
573 "UPDATE roster_deltas SET delta=unbase64(delta);"
574 "UPDATE db_vars SET value=unbase64(value), name=unbase64(name);"
575 "UPDATE public_keys SET keydata=unbase64(keydata);"
576 "UPDATE revision_certs SET value=unbase64(value),"
577 " signature=unbase64(signature);"
578 "UPDATE manifest_certs SET value=unbase64(value),"
579 " signature=unbase64(signature);"
580 "UPDATE revisions SET data=unbase64(data);"
581 "UPDATE branch_epochs SET branch=unbase64(branch);"
582 ;
583
584char const migrate_rosters_no_hash[] =
585 "DROP TABLE rosters;"
586 "DROP TABLE roster_deltas;"
587 "DROP TABLE revision_roster;"
588
589 "CREATE TABLE rosters"
590 " ( id primary key, -- a revision id\n"
591 " checksum not null, -- checksum of 'data', to protect against"
592 " disk corruption\n"
593 " data not null -- compressed, encoded contents of the roster\n"
594 " );"
595
596 "CREATE TABLE roster_deltas"
597 " ( id primary key, -- a revision id\n"
598 " checksum not null, -- checksum of 'delta', to protect against"
599 " disk corruption\n"
600 " base not null, -- joins with either rosters.id or roster_deltas.id\n"
601 " delta not null -- rdiff to construct current from base\n"
602 " );"
603 ;
604
605char const migrate_add_heights[] =
606 "CREATE TABLE heights"
607 " ( revision not null,-- joins with revisions.id\n"
608 " height not null,-- complex height, array of big endian u32 integers\n"
609 " unique(revision, height)"
610 " );"
611 ;
612
613char const migrate_add_heights_index[] =
614 "CREATE INDEX heights__height ON heights (height);"
615 ;
616
617// this is a function because it has to refer to the numeric constant
618// defined in schema_migration.hh.
619static void
620migrate_add_ccode(sqlite3 * db, app_state &)
621{
622 string cmd = "PRAGMA user_version = ";
623 cmd += boost::lexical_cast<string>(mtn_creator_code);
624 sql::exec(db, cmd.c_str());
625}
626
627
628// these must be listed in order so that ones listed earlier override ones
629// listed later
630enum upgrade_regime
631 {
632 upgrade_changesetify,
633 upgrade_rosterify,
634 upgrade_regen_caches,
635 upgrade_none,
636 };
637static void
638dump(enum upgrade_regime const & regime, string & out)
639{
640 switch (regime)
641 {
642 case upgrade_changesetify: out = "upgrade_changesetify"; break;
643 case upgrade_rosterify: out = "upgrade_rosterify"; break;
644 case upgrade_regen_caches: out = "upgrade_regen_caches"; break;
645 case upgrade_none: out = "upgrade_none"; break;
646 default: out = (FL("upgrade_regime(%d)") % regime).str(); break;
647 }
648}
649
650typedef void (*migrator_cb)(sqlite3 *, app_state &);
651
652// Exactly one of migrator_sql and migrator_func should be non-null in
653// all entries in migration_events, except the very last.
654struct migration_event
655{
656 char const * id;
657 char const * migrator_sql;
658 migrator_cb migrator_func;
659 upgrade_regime regime;
660};
661
662// IMPORTANT: whenever you modify this to add a new schema version, you must
663// also add a new migration test for the new schema version. See
664// tests/schema_migration for details.
665
666const migration_event migration_events[] = {
667 { "edb5fa6cef65bcb7d0c612023d267c3aeaa1e57a",
668 migrate_merge_url_and_group, 0, upgrade_none },
669
670 { "f042f3c4d0a4f98f6658cbaf603d376acf88ff4b",
671 migrate_add_hashes_and_merkle_trees, 0, upgrade_none },
672
673 { "8929e54f40bf4d3b4aea8b037d2c9263e82abdf4",
674 migrate_to_revisions, 0, upgrade_changesetify },
675
676 { "c1e86588e11ad07fa53e5d294edc043ce1d4005a",
677 migrate_to_epochs, 0, upgrade_none },
678
679 { "40369a7bda66463c5785d160819ab6398b9d44f4",
680 migrate_to_vars, 0, upgrade_none },
681
682 { "e372b508bea9b991816d1c74680f7ae10d2a6d94",
683 migrate_add_indexes, 0, upgrade_none },
684
685 { "1509fd75019aebef5ac3da3a5edf1312393b70e9",
686 0, migrate_to_external_privkeys, upgrade_none },
687
688 { "bd86f9a90b5d552f0be1fa9aee847ea0f317778b",
689 migrate_add_rosters, 0, upgrade_rosterify },
690
691 { "1db80c7cee8fa966913db1a463ed50bf1b0e5b0e",
692 migrate_files_BLOB, 0, upgrade_none },
693
694 { "9d2b5d7b86df00c30ac34fe87a3c20f1195bb2df",
695 migrate_rosters_no_hash, 0, upgrade_regen_caches },
696
697 { "ae196843d368d042f475e3dadfed11e9d7f9f01e",
698 migrate_add_heights, 0, upgrade_regen_caches },
699
700 { "48fd5d84f1e5a949ca093e87e5ac558da6e5956d",
701 0, migrate_add_ccode, upgrade_none },
702
703 { "fe48b0804e0048b87b4cea51b3ab338ba187bdc2",
704 migrate_add_heights_index, 0, upgrade_none },
705
706 // The last entry in this table should always be the current
707 // schema ID, with 0 for the migrators.
708 { "7ca81b45279403419581d7fde31ed888a80bd34e", 0, 0, upgrade_none }
709};
710const size_t n_migration_events = (sizeof migration_events
711 / sizeof migration_events[0]);
712
713// unfortunately, this has to be aware of the migration_events array and its
714// limits, lest we crash trying to print the garbage on either side.
715static void
716dump(struct migration_event const * const & mref, string & out)
717{
718 struct migration_event const * m = mref;
719 ptrdiff_t i = m - migration_events;
720 if (m == 0)
721 out = "invalid migration event (null pointer)";
722 else if (i < 0 || static_cast<size_t>(i) >= n_migration_events)
723 out = (FL("invalid migration event, index %ld/%lu")
724 % i % n_migration_events).str();
725 else
726 {
727 char const * type;
728 if (m->migrator_sql)
729 type = "SQL only";
730 else if (m->migrator_func)
731 type = "codeful";
732 else
733 type = "none (current)";
734
735 string regime;
736 dump(m->regime, regime);
737
738 out = (FL("migration %ld/%lu: %s, %s, from %s")
739 % i % n_migration_events % type % regime % m->id).str();
740 }
741}
742
743// The next several functions are concerned with calculating the schema hash
744// and determining whether a database is usable (with or without migration).
745static void
746calculate_schema_id(sqlite3 * db, string & ident)
747{
748 sql stmt(db, 1,
749 "SELECT sql FROM sqlite_master "
750 "WHERE (type = 'table' OR type = 'index') "
751 // filter out NULL statements, because
752 // those are auto-generated indices (for
753 // UNIQUE constraints, etc.).
754 "AND sql IS NOT NULL "
755 "AND name not like 'sqlite_stat%' "
756 "ORDER BY name");
757
758 string schema;
759 using boost::char_separator;
760 typedef boost::tokenizer<char_separator<char> > tokenizer;
761 char_separator<char> sep(" \r\n\t", "(),;");
762
763 while (stmt.step())
764 {
765 string table_schema(stmt.column_string(0));
766 tokenizer tokens(table_schema, sep);
767 for (tokenizer::iterator i = tokens.begin(); i != tokens.end(); i++)
768 {
769 if (schema.size() != 0)
770 schema += " ";
771 schema += *i;
772 }
773 }
774
775 uint32_t code = sql::value(db, "PRAGMA user_version");
776 if (code != 0)
777 {
778 schema += " PRAGMA user_version = ";
779 schema += boost::lexical_cast<string>(code);
780 }
781 hexenc<id> tid;
782 calculate_ident(data(schema), tid);
783 ident = tid();
784}
785
786// Look through the migration_events table and return a pointer to the entry
787// corresponding to database DB, or null if it isn't there (i.e. if the
788// database schema is not one we know).
789static migration_event const *
790find_migration(sqlite3 * db)
791{
792 string id;
793 calculate_schema_id(db, id);
794
795 for (migration_event const *m = migration_events + n_migration_events - 1;
796 m >= migration_events; m--)
797 if (m->id == id)
798 return m;
799
800 return 0;
801}
802
803// This enumerates the possible mismatches between the monotone executable
804// and its database.
805enum schema_mismatch_case
806 {
807 SCHEMA_MATCHES = 0,
808 SCHEMA_MIGRATION_NEEDED,
809 SCHEMA_TOO_NEW,
810 SCHEMA_NOT_MONOTONE,
811 SCHEMA_EMPTY
812 };
813static void dump(schema_mismatch_case const & cat, std::string & out)
814{
815 switch (cat)
816 {
817 case SCHEMA_MATCHES: out = "SCHEMA_MATCHES"; break;
818 case SCHEMA_MIGRATION_NEEDED: out = "SCHEMA_MIGRATION_NEEDED"; break;
819 case SCHEMA_TOO_NEW: out = "SCHEMA_TOO_NEW"; break;
820 case SCHEMA_NOT_MONOTONE: out = "SCHEMA_NOT_MONOTONE"; break;
821 case SCHEMA_EMPTY: out = "SCHEMA_EMPTY"; break;
822 default: out = (FL("schema_mismatch_case(%d)") % cat).str(); break;
823 }
824}
825
826
827static schema_mismatch_case
828classify_schema(sqlite3 * db, migration_event const * m = 0)
829{
830 if (!m)
831 m = find_migration(db);
832
833 if (m)
834 {
835 if (m->migrator_sql || m->migrator_func)
836 return SCHEMA_MIGRATION_NEEDED;
837 else
838 return SCHEMA_MATCHES;
839 }
840 else
841 {
842 // Distinguish an utterly empty database, such as is created by
843 // "mtn db load < /dev/null", or by the sqlite3 command line utility
844 // if you don't give it anything to do.
845 if (sql::value(db, "SELECT COUNT(*) FROM sqlite_master") == 0)
846 return SCHEMA_EMPTY;
847
848 // monotone started setting this value in database headers only with
849 // version 0.33, but all previous versions' databases are recognized
850 // by their schema hashes.
851
852 uint32_t code = sql::value(db, "PRAGMA user_version");
853 if (code != mtn_creator_code)
854 return SCHEMA_NOT_MONOTONE;
855
856 return SCHEMA_TOO_NEW;
857 }
858}
859
860string
861describe_sql_schema(sqlite3 * db)
862{
863 I(db != NULL);
864 string hash;
865 calculate_schema_id(db, hash);
866
867 switch (classify_schema(db))
868 {
869 case SCHEMA_MATCHES:
870 return (F("%s (usable)") % hash).str();
871 case SCHEMA_MIGRATION_NEEDED:
872 return (F("%s (migration needed)") % hash).str();
873 case SCHEMA_TOO_NEW:
874 return (F("%s (too new, cannot use)") % hash).str();
875 case SCHEMA_NOT_MONOTONE:
876 return (F("%s (not a monotone database)") % hash).str();
877 case SCHEMA_EMPTY:
878 return (F("%s (database has no tables!)") % hash).str();
879 default:
880 I(false);
881 }
882}
883
884// Provide sensible diagnostics for a database schema whose hash we do not
885// recognize. (Shared between check_sql_schema and migrate_sql_schema.)
886static void
887diagnose_unrecognized_schema(schema_mismatch_case cat,
888 system_path const & filename)
889{
890 N(cat != SCHEMA_EMPTY,
891 F("cannot use the empty sqlite database %s\n"
892 "(monotone databases must be created with '%s db init')")
893 % filename % ui.prog_name);
894
895 N(cat != SCHEMA_NOT_MONOTONE,
896 F("%s does not appear to be a monotone database\n")
897 % filename);
898
899 N(cat != SCHEMA_TOO_NEW,
900 F("%s appears to be a monotone database, but this version of\n"
901 "monotone does not recognize its schema.\n"
902 "you probably need a newer version of monotone.")
903 % filename);
904}
905
906// check_sql_schema is called by database.cc on open, to determine whether
907// the schema is up to date. If it returns at all, the schema is indeed
908// up to date (otherwise it throws a diagnostic).
909void
910check_sql_schema(sqlite3 * db, system_path const & filename)
911{
912 I(db != NULL);
913
914 schema_mismatch_case cat = classify_schema(db);
915
916 diagnose_unrecognized_schema(cat, filename);
917
918 N(cat != SCHEMA_MIGRATION_NEEDED,
919 F("database %s is laid out according to an old schema\n"
920 "try '%s db migrate' to upgrade\n"
921 "(this is irreversible; you may want to make a backup copy first)")
922 % filename % ui.prog_name);
923}
924
925void
926migrate_sql_schema(sqlite3 * db, app_state & app)
927{
928 I(db != NULL);
929
930 upgrade_regime regime = upgrade_none; MM(regime);
931
932 // Take an exclusive lock on the database before we try to read anything
933 // from it. If we don't take this lock until the beginning of the
934 // "migrating data" phase, two simultaneous "db migrate" processes could
935 // race through the "calculating migration" phase; then one of them would
936 // wait for the other to finish all the migration steps, and trip over the
937 // invariant check inside the for loop.
938 {
939 transaction guard(db);
940
941 P(F("calculating migration..."));
942
943 migration_event const *m; MM(m);
944 schema_mismatch_case cat; MM(cat);
945 m = find_migration(db);
946 cat = classify_schema(db, m);
947
948 diagnose_unrecognized_schema(cat, app.db.get_filename());
949
950 // We really want 'db migrate' on an up-to-date schema to be a no-op
951 // (no vacuum or anything, even), so that automated scripts can fire
952 // one off optimistically and not have to worry about getting their
953 // administrators to do it by hand.
954 if (cat == SCHEMA_MATCHES)
955 {
956 P(F("no migration performed; database schema already up-to-date"));
957 return;
958 }
959
960 sql::create_function(db, "sha1", sqlite_sha1_fn);
961 sql::create_function(db, "unbase64", sqlite3_unbase64_fn);
962
963 P(F("migrating data..."));
964
965 for (;;)
966 {
967 // confirm that we are where we ought to be
968 string id; MM(id);
969 calculate_schema_id(db, id);
970
971 I(id == m->id);
972 I(!m->migrator_sql || !m->migrator_func);
973
974 if (m->migrator_sql)
975 sql::exec(db, m->migrator_sql);
976 else if (m->migrator_func)
977 m->migrator_func(db, app);
978 else
979 break;
980
981 regime = std::min(regime, m->regime);
982
983 m++;
984 I(m < migration_events + n_migration_events);
985 P(F("migrated to schema %s") % m->id);
986 }
987
988 P(F("committing changes to database"));
989 guard.commit();
990 }
991
992 P(F("optimizing database"));
993 sql::exec(db, "VACUUM");
994
995 switch (regime)
996 {
997 case upgrade_changesetify:
998 case upgrade_rosterify:
999 {
1000 string command_str = (regime == upgrade_changesetify
1001 ? "changesetify" : "rosterify");
1002 P(F("NOTE: because this database was last used by a rather old version\n"
1003 "of monotone, you're not done yet. If you're a project leader, then\n"
1004 "see the file UPGRADE for instructions on running '%s db %s'")
1005 % ui.prog_name % command_str);
1006 }
1007 break;
1008 case upgrade_regen_caches:
1009 P(F("NOTE: this upgrade cleared monotone's caches\n"
1010 "you should now run '%s db regenerate_caches'")
1011 % ui.prog_name);
1012 break;
1013 case upgrade_none:
1014 break;
1015 }
1016}
1017
1018// test_migration_step runs the migration step from SCHEMA to its successor,
1019// *without* validating that the database actually conforms to that schema
1020// first. the point of this is to test error recovery from conditions that
1021// are not accessible through normal malformed dumps (because the schema
1022// conformance check will reject them).
1023
1024void
1025test_migration_step(sqlite3 * db, app_state & app, string const & schema)
1026{
1027 I(db != NULL);
1028 sql::create_function(db, "sha1", sqlite_sha1_fn);
1029 sql::create_function(db, "unbase64", sqlite3_unbase64_fn);
1030
1031 transaction guard(db);
1032
1033 migration_event const *m;
1034 for (m = migration_events + n_migration_events - 1;
1035 m >= migration_events; m--)
1036 if (schema == m->id)
1037 break;
1038
1039 N(m >= migration_events,
1040 F("cannot test migration from unknown schema %s") % schema);
1041
1042 N(m->migrator_sql || m->migrator_func,
1043 F("schema %s is up to date") % schema);
1044
1045 L(FL("testing migration from %s to %s\n in database %s")
1046 % schema % m[1].id % app.db.get_filename());
1047
1048 if (m->migrator_sql)
1049 sql::exec(db, m->migrator_sql);
1050 else
1051 m->migrator_func(db, app);
1052
1053 // in the unlikely event that we get here ...
1054 P(F("successful migration to schema %s") % m[1].id);
1055 guard.commit();
1056}
1057
1058
1059// Local Variables:
1060// mode: C++
1061// fill-column: 76
1062// c-file-style: "gnu"
1063// indent-tabs-mode: nil
1064// End:
1065// vim: et:sw=2:sts=2:ts=2:cino=>2s,{s,\:s,+s,t0,g0,^-2,e-2,n-2,p2s,(0,=s:

Archive Download this file

Branches

Tags

Quick Links:     www.monotone.ca    -     Downloads    -     Documentation    -     Wiki    -     Code Forge    -     Build Status