monotone

monotone Mtn Source Tree

Root/schema_migration.cc

1// copyright (C) 2002, 2003 graydon hoare <graydon@pobox.com>
2// all rights reserved.
3// licensed to the public under the terms of the GNU GPL (>= 2)
4// see the file COPYING for details
5
6#include <algorithm>
7#include <string>
8#include <vector>
9#include <locale>
10#include <stdexcept>
11#include <iostream>
12
13#include <boost/tokenizer.hpp>
14
15#include "schema_migration.hh"
16#include "sqlite/sqlite.h"
17#include "cryptopp/filters.h"
18#include "cryptopp/sha.h"
19#include "cryptopp/hex.h"
20
21// this file knows how to migrate schema databases. the general strategy is
22// to hash each schema we ever use, and make a list of the SQL commands
23// required to get from one hash value to the next. when you do a
24// migration, the migrator locates your current db's state on the list and
25// then runs all the migration functions between that point and the target
26// of the migration.
27
28// you will notice a little bit of duplicated code between here and
29// transforms.cc / database.cc; this is to facilitate includion of
30// migration capability into the depot code, which does not link against
31// those objects.
32
33using namespace std;
34
35typedef boost::tokenizer<boost::char_separator<char> > tokenizer;
36
37static string lowercase(string const & in)
38{
39 size_t const sz = in.size();
40 char buf[sz];
41 in.copy(buf, sz);
42 use_facet< ctype<char> >(locale::locale()).tolower(buf, buf+sz);
43 return string(buf,sz);
44}
45
46static void massage_sql_tokens(string const & in,
47 string & out)
48{
49 boost::char_separator<char> sep(" \r\n\t", "(),;");
50 tokenizer tokens(in, sep);
51 out.clear();
52 for (tokenizer::iterator i = tokens.begin();
53 i != tokens.end(); ++i)
54 {
55 if (i != tokens.begin())
56out += " ";
57 out += *i;
58 }
59}
60
61static void calculate_id(string const & in,
62 string & ident)
63{
64 CryptoPP::SHA hash;
65 unsigned int const sz = 2 * CryptoPP::SHA::DIGESTSIZE;
66 char buffer[sz];
67 CryptoPP::StringSource
68 s(in, true, new CryptoPP::HashFilter
69 (hash, new CryptoPP::HexEncoder
70 (new CryptoPP::ArraySink(reinterpret_cast<byte *>(buffer), sz))));
71 ident = lowercase(string(buffer, sz));
72}
73
74
75static void sqlite_sha1_fn(sqlite_func *f, int nargs, char const ** args)
76{
77 string tmp, sha;
78 if (nargs <= 1)
79 {
80 sqlite_set_result_error(f, "need at least 1 arg to sha1()", -1);
81 return;
82 }
83
84 if (nargs == 1)
85 tmp = string(args[0]);
86 else
87 {
88 string sep = string(args[0]);
89 tmp = string(args[1]);
90 for (int i = 2; i < nargs; ++i)
91{
92 tmp += sep + string(args[i]);
93}
94 }
95 calculate_id(tmp, sha);
96 sqlite_set_result_string(f,sha.c_str(),sha.size());
97}
98
99int append_sql_stmt(void * vp,
100 int ncols,
101 char ** values,
102 char ** colnames)
103{
104 if (ncols != 1)
105 return 1;
106
107 if (vp == NULL)
108 return 1;
109
110 if (values == NULL)
111 return 1;
112
113 if (values[0] == NULL)
114 return 1;
115
116 string *str = reinterpret_cast<string *>(vp);
117 str->append(values[0]);
118 str->append("\n");
119 return 0;
120}
121
122void calculate_schema_id(sqlite *sql, string & id)
123{
124 id.clear();
125 string tmp, tmp2;
126 int res = sqlite_exec_printf(sql,
127 "SELECT sql FROM sqlite_master "
128 "WHERE type = 'table' "
129 "ORDER BY name",
130 &append_sql_stmt, &tmp, NULL);
131 if (res != SQLITE_OK)
132 {
133 sqlite_exec(sql, "ROLLBACK", NULL, NULL, NULL);
134 throw runtime_error("failure extracting schema from sqlite_master");
135 }
136 massage_sql_tokens(tmp, tmp2);
137 calculate_id(tmp2, id);
138}
139
140typedef bool (*migrator_cb)(sqlite *, char **);
141
142struct migrator
143{
144 vector< pair<string,migrator_cb> > migration_events;
145
146 void add(string schema_id, migrator_cb cb)
147 {
148 migration_events.push_back(make_pair(schema_id, cb));
149 }
150
151 void migrate(sqlite *sql, string target_id)
152 {
153 string init;
154 calculate_schema_id(sql, init);
155
156 if (sql == NULL)
157 throw runtime_error("NULL sqlite object given to migrate");
158
159 if (sqlite_create_function(sql, "sha1", -1, &sqlite_sha1_fn, NULL))
160 throw runtime_error("error registering sha1 function with sqlite");
161
162 bool migrating = false;
163 for (vector< pair<string, migrator_cb> >::const_iterator i = migration_events.begin();
164 i != migration_events.end(); ++i)
165 {
166
167if (i->first == init)
168 {
169 if (sqlite_exec(sql, "BEGIN", NULL, NULL, NULL) != SQLITE_OK)
170 throw runtime_error("error at transaction BEGIN statement");
171 migrating = true;
172 }
173
174if (migrating)
175 {
176 // confirm that we are where we ought to be
177 string curr;
178 char *errmsg = NULL;
179 calculate_schema_id(sql, curr);
180 if (curr != i->first)
181 {
182if (migrating)
183 sqlite_exec(sql, "ROLLBACK", NULL, NULL, NULL);
184throw runtime_error("mismatched pre-state to migration step");
185 }
186
187 if (i->second == NULL)
188 {
189sqlite_exec(sql, "ROLLBACK", NULL, NULL, NULL);
190throw runtime_error("NULL migration specifier");
191 }
192
193 // do this migration step
194 else if (! i->second(sql, &errmsg))
195 {
196string e("migration step failed");
197if (errmsg != NULL)
198 e.append(string(": ") + errmsg);
199sqlite_exec(sql, "ROLLBACK", NULL, NULL, NULL);
200throw runtime_error(e);
201 }
202 }
203 }
204
205 // confirm that our target schema was met
206 if (migrating)
207 {
208string curr;
209calculate_schema_id(sql, curr);
210if (curr != target_id)
211 {
212 sqlite_exec(sql, "ROLLBACK", NULL, NULL, NULL);
213 throw runtime_error("mismatched result of migration, "
214"got " + curr + ", wanted " + target_id);
215 }
216if (sqlite_exec(sql, "COMMIT", NULL, NULL, NULL) != SQLITE_OK)
217 {
218 throw runtime_error("failure on COMMIT");
219 }
220 }
221 }
222};
223
224static bool move_table(sqlite *sql, char **errmsg,
225 char const * srcname,
226 char const * dstname,
227 char const * dstschema)
228{
229 int res =
230 sqlite_exec_printf(sql, "CREATE TABLE %s %s", NULL, NULL, errmsg,
231 dstname, dstschema);
232 if (res != SQLITE_OK)
233 return false;
234
235 res =
236 sqlite_exec_printf(sql, "INSERT INTO %s SELECT * FROM %s",
237 NULL, NULL, errmsg, dstname, srcname);
238 if (res != SQLITE_OK)
239 return false;
240
241 res =
242 sqlite_exec_printf(sql, "DROP TABLE %s",
243 NULL, NULL, errmsg, srcname);
244 if (res != SQLITE_OK)
245 return false;
246
247 return true;
248}
249
250
251static bool migrate_depot_split_seqnumbers_into_groups(sqlite * sql,
252 char ** errmsg)
253{
254
255 // this migration event handles the bug related to sequence numbers only
256 // being assigned on a per-depot, rather than per-group basis. in the
257 // process it also corrects the false UNIQUE constraint on contents
258 // (which may occur multiple times in different groups).
259 //
260 // after this migration event, all major sequence numbers are bumped, so
261 // your clients will re-fetch (idempotently) the contents of your depot.
262
263 if (!move_table(sql, errmsg,
264 "packets",
265 "tmp",
266 "("
267 "major INTEGER,"
268 "minor INTEGER,"
269 "groupname TEXT NOT NULL,"
270 "adler32 TEXT NOT NULL,"
271 "contents TEXT NOT NULL UNIQUE,"
272 "unique(major, minor)"
273 ")"))
274 return false;
275
276 int res =
277 sqlite_exec(sql,
278" UPDATE tmp SET major = \n"
279" (SELECT MAX(major) + 1 FROM tmp);\n",
280NULL, NULL, errmsg);
281 if (res != SQLITE_OK)
282 return false;
283
284 if (!move_table(sql, errmsg,
285 "tmp",
286 "packets",
287 "("
288 "major INTEGER,"
289 "minor INTEGER,"
290 "groupname TEXT NOT NULL,"
291 "adler32 TEXT NOT NULL,"
292 "contents TEXT NOT NULL,"
293 "unique(groupname, contents),"
294 "unique(major, minor, groupname)"
295 ")"))
296 return false;
297
298 return true;
299}
300
301static bool migrate_depot_make_seqnumbers_non_null(sqlite * sql,
302 char ** errmsg)
303{
304 // this just adds NOT NULL constraints to the INTEGER fields
305
306 if (!move_table(sql, errmsg,
307 "packets",
308 "tmp",
309 "("
310 "major INTEGER,"
311 "minor INTEGER,"
312 "groupname TEXT NOT NULL,"
313 "adler32 TEXT NOT NULL,"
314 "contents TEXT NOT NULL,"
315 "unique(groupname, contents),"
316 "unique(major, minor, groupname)"
317 ")"))
318 return false;
319
320 if (!move_table(sql, errmsg,
321 "tmp",
322 "packets",
323 "("
324 "major INTEGER NOT NULL,"
325 "minor INTEGER NOT NULL,"
326 "groupname TEXT NOT NULL,"
327 "adler32 TEXT NOT NULL,"
328 "contents TEXT NOT NULL,"
329 "unique(groupname, contents),"
330 "unique(major, minor, groupname)"
331 ")"))
332 return false;
333
334 return true;
335}
336
337
338static bool migrate_client_merge_url_and_group(sqlite * sql,
339 char ** errmsg)
340{
341
342 // migrate the posting_queue table
343 if (!move_table(sql, errmsg,
344 "posting_queue",
345 "tmp",
346 "("
347 "url not null,"
348 "groupname not null,"
349 "content not null"
350 ")"))
351 return false;
352
353 int res = sqlite_exec_printf(sql, "CREATE TABLE posting_queue "
354 "("
355 "url not null, -- URL we are going to send this to\n"
356 "content not null -- the packets we're going to send\n"
357 ")", NULL, NULL, errmsg);
358 if (res != SQLITE_OK)
359 return false;
360
361 res = sqlite_exec_printf(sql, "INSERT INTO posting_queue "
362 "SELECT "
363 "(url || '/' || groupname), "
364 "content "
365 "FROM tmp", NULL, NULL, errmsg);
366 if (res != SQLITE_OK)
367 return false;
368
369 res = sqlite_exec_printf(sql, "DROP TABLE tmp", NULL, NULL, errmsg);
370 if (res != SQLITE_OK)
371 return false;
372
373
374 // migrate the incoming_queue table
375 if (!move_table(sql, errmsg,
376 "incoming_queue",
377 "tmp",
378 "("
379 "url not null,"
380 "groupname not null,"
381 "content not null"
382 ")"))
383 return false;
384
385 res = sqlite_exec_printf(sql, "CREATE TABLE incoming_queue "
386 "("
387 "url not null, -- URL we got this bundle from\n"
388 "content not null -- the packets we're going to read\n"
389 ")", NULL, NULL, errmsg);
390 if (res != SQLITE_OK)
391 return false;
392
393 res = sqlite_exec_printf(sql, "INSERT INTO incoming_queue "
394 "SELECT "
395 "(url || '/' || groupname), "
396 "content "
397 "FROM tmp", NULL, NULL, errmsg);
398 if (res != SQLITE_OK)
399 return false;
400
401 res = sqlite_exec_printf(sql, "DROP TABLE tmp", NULL, NULL, errmsg);
402 if (res != SQLITE_OK)
403 return false;
404
405
406 // migrate the sequence_numbers table
407 if (!move_table(sql, errmsg,
408 "sequence_numbers",
409 "tmp",
410 "("
411 "url not null,"
412 "groupname not null,"
413 "major not null,"
414 "minor not null,"
415 "unique(url, groupname)"
416 ")"
417 ))
418 return false;
419
420 res = sqlite_exec_printf(sql, "CREATE TABLE sequence_numbers "
421 "("
422 "url primary key, -- URL to read from\n"
423 "major not null, -- 0 in news servers, may be higher in depots\n"
424 "minor not null -- last article / packet sequence number we got\n"
425 ")", NULL, NULL, errmsg);
426 if (res != SQLITE_OK)
427 return false;
428
429 res = sqlite_exec_printf(sql, "INSERT INTO sequence_numbers "
430 "SELECT "
431 "(url || '/' || groupname), "
432 "major, "
433 "minor "
434 "FROM tmp", NULL, NULL, errmsg);
435 if (res != SQLITE_OK)
436 return false;
437
438 res = sqlite_exec_printf(sql, "DROP TABLE tmp", NULL, NULL, errmsg);
439 if (res != SQLITE_OK)
440 return false;
441
442
443 // migrate the netserver_manifests table
444 if (!move_table(sql, errmsg,
445 "netserver_manifests",
446 "tmp",
447 "("
448 "url not null,"
449 "groupname not null,"
450 "manifest not null,"
451 "unique(url, groupname, manifest)"
452 ")"
453 ))
454 return false;
455
456 res = sqlite_exec_printf(sql, "CREATE TABLE netserver_manifests "
457 "("
458 "url not null, -- url of some server\n"
459 "manifest not null, -- manifest which exists on url\n"
460 "unique(url, manifest)"
461 ")", NULL, NULL, errmsg);
462 if (res != SQLITE_OK)
463 return false;
464
465 res = sqlite_exec_printf(sql, "INSERT INTO netserver_manifests "
466 "SELECT "
467 "(url || '/' || groupname), "
468 "manifest "
469 "FROM tmp", NULL, NULL, errmsg);
470 if (res != SQLITE_OK)
471 return false;
472
473 res = sqlite_exec_printf(sql, "DROP TABLE tmp", NULL, NULL, errmsg);
474 if (res != SQLITE_OK)
475 return false;
476
477 return true;
478}
479
480static bool migrate_client_add_hashes_and_merkle_trees(sqlite * sql,
481 char ** errmsg)
482{
483
484 // add the column to manifest_certs
485 if (!move_table(sql, errmsg,
486 "manifest_certs",
487 "tmp",
488 "("
489 "id not null,"
490 "name not null,"
491 "value not null,"
492 "keypair not null,"
493 "signature not null,"
494 "unique(name, id, value, keypair, signature)"
495 ")"))
496 return false;
497
498 int res = sqlite_exec_printf(sql, "CREATE TABLE manifest_certs\n"
499 "(\n"
500 "hash not null unique, -- hash of remaining fields separated by \":\"\n"
501 "id not null, -- joins with manifests.id or manifest_deltas.id\n"
502 "name not null, -- opaque string chosen by user\n"
503 "value not null, -- opaque blob\n"
504 "keypair not null, -- joins with public_keys.id\n"
505 "signature not null, -- RSA/SHA1 signature of \"[name@id:val]\"\n"
506 "unique(name, id, value, keypair, signature)\n"
507 ")", NULL, NULL, errmsg);
508 if (res != SQLITE_OK)
509 return false;
510
511 res = sqlite_exec_printf(sql, "INSERT INTO manifest_certs "
512 "SELECT "
513 "sha1(':', id, name, value, keypair, signature), "
514 "id, name, value, keypair, signature "
515 "FROM tmp", NULL, NULL, errmsg);
516 if (res != SQLITE_OK)
517 return false;
518
519 res = sqlite_exec_printf(sql, "DROP TABLE tmp", NULL, NULL, errmsg);
520 if (res != SQLITE_OK)
521 return false;
522
523 // add the column to file_certs
524 if (!move_table(sql, errmsg,
525 "file_certs",
526 "tmp",
527 "("
528 "id not null,"
529 "name not null,"
530 "value not null,"
531 "keypair not null,"
532 "signature not null,"
533 "unique(name, id, value, keypair, signature)"
534 ")"))
535 return false;
536
537 res = sqlite_exec_printf(sql, "CREATE TABLE file_certs\n"
538 "(\n"
539 "hash not null unique, -- hash of remaining fields separated by \":\"\n"
540 "id not null, -- joins with files.id or file_deltas.id\n"
541 "name not null, -- opaque string chosen by user\n"
542 "value not null, -- opaque blob\n"
543 "keypair not null, -- joins with public_keys.id\n"
544 "signature not null, -- RSA/SHA1 signature of \"[name@id:val]\"\n"
545 "unique(name, id, value, keypair, signature)\n"
546 ")", NULL, NULL, errmsg);
547 if (res != SQLITE_OK)
548 return false;
549
550 res = sqlite_exec_printf(sql, "INSERT INTO file_certs "
551 "SELECT "
552 "sha1(':', id, name, value, keypair, signature), "
553 "id, name, value, keypair, signature "
554 "FROM tmp", NULL, NULL, errmsg);
555 if (res != SQLITE_OK)
556 return false;
557
558 res = sqlite_exec_printf(sql, "DROP TABLE tmp", NULL, NULL, errmsg);
559 if (res != SQLITE_OK)
560 return false;
561
562 // add the column to public_keys
563 if (!move_table(sql, errmsg,
564 "public_keys",
565 "tmp",
566 "("
567 "id primary key,"
568 "keydata not null"
569 ")"))
570 return false;
571
572 res = sqlite_exec_printf(sql, "CREATE TABLE public_keys\n"
573 "(\n"
574 "hash not null unique, -- hash of remaining fields separated by \":\"\n"
575 "id primary key, -- key identifier chosen by user\n"
576 "keydata not null -- RSA public params\n"
577 ")", NULL, NULL, errmsg);
578 if (res != SQLITE_OK)
579 return false;
580
581 res = sqlite_exec_printf(sql, "INSERT INTO public_keys "
582 "SELECT "
583 "sha1(':', id, keydata), "
584 "id, keydata "
585 "FROM tmp", NULL, NULL, errmsg);
586 if (res != SQLITE_OK)
587 return false;
588
589 res = sqlite_exec_printf(sql, "DROP TABLE tmp", NULL, NULL, errmsg);
590 if (res != SQLITE_OK)
591 return false;
592
593 // add the column to private_keys
594 if (!move_table(sql, errmsg,
595 "private_keys",
596 "tmp",
597 "("
598 "id primary key,"
599 "keydata not null"
600 ")"))
601 return false;
602
603 res = sqlite_exec_printf(sql, "CREATE TABLE private_keys\n"
604 "(\n"
605 "hash not null unique, -- hash of remaining fields separated by \":\"\n"
606 "id primary key, -- as in public_keys (same identifiers, in fact)\n"
607 "keydata not null -- encrypted RSA private params\n"
608 ")", NULL, NULL, errmsg);
609 if (res != SQLITE_OK)
610 return false;
611
612 res = sqlite_exec_printf(sql, "INSERT INTO private_keys "
613 "SELECT "
614 "sha1(':', id, keydata), "
615 "id, keydata "
616 "FROM tmp", NULL, NULL, errmsg);
617 if (res != SQLITE_OK)
618 return false;
619
620 res = sqlite_exec_printf(sql, "DROP TABLE tmp", NULL, NULL, errmsg);
621 if (res != SQLITE_OK)
622 return false;
623
624 // add the merkle tree stuff
625
626 res = sqlite_exec_printf(sql,
627 "CREATE TABLE merkle_nodes\n"
628 "(\n"
629 "type not null, -- \"key\", \"mcert\", \"fcert\", \"manifest\"\n"
630 "collection not null, -- name chosen by user\n"
631 "level not null, -- tree level this prefix encodes\n"
632 "prefix not null, -- label identifying node in tree\n"
633 "body not null, -- binary, base64'ed node contents\n"
634 "unique(type, collection, level, prefix)\n"
635 ")", NULL, NULL, errmsg);
636 if (res != SQLITE_OK)
637 return false;
638
639 return true;
640}
641
642void migrate_depot_schema(sqlite *sql)
643{
644 migrator m;
645
646 m.add("da3d5798a6ae61bd6566e74d8888faebc413dd2f",
647&migrate_depot_split_seqnumbers_into_groups);
648
649 m.add("b820522b75efb31c8afb1b6f114841354d87e22d",
650&migrate_depot_make_seqnumbers_non_null);
651
652 m.migrate(sql, "b0f3041a8ded95006584340ef76bd70ae81bb376");
653
654 if (sqlite_exec(sql, "VACUUM", NULL, NULL, NULL) != SQLITE_OK)
655 throw runtime_error("error vacuuming after migration");
656}
657
658void migrate_monotone_schema(sqlite *sql)
659{
660
661 migrator m;
662
663 m.add("edb5fa6cef65bcb7d0c612023d267c3aeaa1e57a",
664&migrate_client_merge_url_and_group);
665
666 m.add("f042f3c4d0a4f98f6658cbaf603d376acf88ff4b",
667&migrate_client_add_hashes_and_merkle_trees);
668
669 m.migrate(sql, "8929e54f40bf4d3b4aea8b037d2c9263e82abdf4");
670
671 if (sqlite_exec(sql, "VACUUM", NULL, NULL, NULL) != SQLITE_OK)
672 throw runtime_error("error vacuuming after migration");
673
674}

Archive Download this file

Branches

Tags

Quick Links:     www.monotone.ca    -     Downloads    -     Documentation    -     Wiki    -     Code Forge    -     Build Status