monotone

monotone Mtn Source Tree

Root/database.cc

1// Copyright (C) 2002 Graydon Hoare <graydon@pobox.com>
2//
3// This program is made available under the GNU GPL version 2.0 or
4// greater. See the accompanying file COPYING for details.
5//
6// This program is distributed WITHOUT ANY WARRANTY; without even the
7// implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
8// PURPOSE.
9
10#include "base.hh"
11#include <algorithm>
12#include <deque>
13#include <fstream>
14#include <iterator>
15#include <list>
16#include <set>
17#include <sstream>
18#include "vector.hh"
19
20#include <string.h>
21
22#include <boost/shared_ptr.hpp>
23#include "lexical_cast.hh"
24
25#include "sqlite/sqlite3.h"
26
27#include "app_state.hh"
28#include "cert.hh"
29#include "cleanup.hh"
30#include "constants.hh"
31#include "dates.hh"
32#include "database.hh"
33#include "hash_map.hh"
34#include "keys.hh"
35#include "platform-wrapped.hh"
36#include "revision.hh"
37#include "safe_map.hh"
38#include "sanity.hh"
39#include "schema_migration.hh"
40#include "transforms.hh"
41#include "ui.hh"
42#include "vocab.hh"
43#include "vocab_cast.hh"
44#include "xdelta.hh"
45#include "epoch.hh"
46#include "graph.hh"
47#include "roster.hh"
48#include "roster_delta.hh"
49#include "rev_height.hh"
50#include "vocab_hash.hh"
51#include "globish.hh"
52#include "work.hh"
53#include "lua_hooks.hh"
54#include "outdated_indicator.hh"
55#include "lru_writeback_cache.hh"
56
57#include "botan/botan.h"
58#include "botan/rsa.h"
59#include "botan/keypair.h"
60#include "botan/pem.h"
61
62// defined in schema.c, generated from schema.sql:
63extern char const schema_constant[];
64
65// this file defines a public, typed interface to the database.
66// the database class encapsulates all knowledge about sqlite,
67// the schema, and all SQL statements used to access the schema.
68//
69// see file schema.sql for the text of the schema.
70
71using std::deque;
72using std::istream;
73using std::ifstream;
74using std::make_pair;
75using std::map;
76using std::multimap;
77using std::ostream;
78using std::pair;
79using std::set;
80using std::string;
81using std::vector;
82
83using boost::shared_ptr;
84using boost::shared_dynamic_cast;
85using boost::lexical_cast;
86
87using Botan::PK_Encryptor;
88using Botan::PK_Verifier;
89using Botan::SecureVector;
90using Botan::X509_PublicKey;
91using Botan::RSA_PublicKey;
92
93int const one_row = 1;
94int const one_col = 1;
95int const any_rows = -1;
96int const any_cols = -1;
97
98namespace
99{
100 struct query_param
101 {
102 enum arg_type { text, blob };
103 arg_type type;
104 string data;
105 };
106
107 query_param
108 text(string const & txt)
109 {
110 query_param q = {
111 query_param::text,
112 txt,
113 };
114 return q;
115 }
116
117 query_param
118 blob(string const & blb)
119 {
120 query_param q = {
121 query_param::blob,
122 blb,
123 };
124 return q;
125 }
126
127 struct query
128 {
129 explicit query(string const & cmd)
130 : sql_cmd(cmd)
131 {}
132
133 query()
134 {}
135
136 query & operator %(query_param const & qp)
137 {
138 args.push_back(qp);
139 return *this;
140 }
141
142 vector<query_param> args;
143 string sql_cmd;
144 };
145
146 typedef vector< vector<string> > results;
147
148 struct statement
149 {
150 statement() : count(0), stmt(0, sqlite3_finalize) {}
151 int count;
152 cleanup_ptr<sqlite3_stmt*, int> stmt;
153 };
154
155 struct roster_size_estimator
156 {
157 unsigned long operator() (cached_roster const & cr)
158 {
159 I(cr.first);
160 I(cr.second);
161 // do estimate using a totally made up multiplier, probably wildly off
162 return (cr.first->all_nodes().size()
163 * constants::db_estimated_roster_node_sz);
164 }
165 };
166
167 struct datasz
168 {
169 unsigned long operator()(data const & t) { return t().size(); }
170 };
171
172 enum open_mode { normal_mode = 0,
173 schema_bypass_mode,
174 format_bypass_mode };
175
176 typedef hashmap::hash_map<revision_id, set<revision_id> > parent_id_map;
177 typedef hashmap::hash_map<revision_id, rev_height> height_map;
178
179 typedef hashmap::hash_map<rsa_keypair_id,
180 pair<shared_ptr<Botan::PK_Verifier>,
181 shared_ptr<Botan::RSA_PublicKey> >
182 > verifier_cache;
183
184} // anonymous namespace
185
186class database_impl
187{
188 friend class database;
189
190 // for scoped_ptr's sake
191public:
192 explicit database_impl(system_path const &);
193 ~database_impl();
194
195private:
196
197 //
198 // --== Opening the database and schema checking ==--
199 //
200 system_path const filename;
201 struct sqlite3 * __sql;
202
203 void install_functions();
204 struct sqlite3 * sql(enum open_mode mode = normal_mode);
205
206 void check_filename();
207 void check_db_exists();
208 void check_db_nonexistent();
209 void open();
210 void close();
211 void check_format();
212
213 //
214 // --== Basic SQL interface and statement caching ==--
215 //
216 map<string, statement> statement_cache;
217
218 void fetch(results & res,
219 int const want_cols, int const want_rows,
220 query const & q);
221 void execute(query const & q);
222
223 bool table_has_entry(id const & key, string const & column,
224 string const & table);
225
226 //
227 // --== Generic database metadata gathering ==--
228 //
229 string count(string const & table);
230 string space(string const & table,
231 string const & concatenated_columns,
232 u64 & total);
233 unsigned int page_size();
234 unsigned int cache_size();
235
236 //
237 // --== Transactions ==--
238 //
239 int transaction_level;
240 bool transaction_exclusive;
241 void begin_transaction(bool exclusive);
242 void commit_transaction();
243 void rollback_transaction();
244 friend class conditional_transaction_guard;
245
246 struct roster_writeback_manager
247 {
248 database_impl & imp;
249 roster_writeback_manager(database_impl & imp) : imp(imp) {}
250 void writeout(revision_id const &, cached_roster const &);
251 };
252 LRUWritebackCache<revision_id, cached_roster,
253 roster_size_estimator, roster_writeback_manager>
254 roster_cache;
255
256 bool have_delayed_file(file_id const & id);
257 void load_delayed_file(file_id const & id, file_data & dat);
258 void cancel_delayed_file(file_id const & id);
259 void drop_or_cancel_file(file_id const & id);
260 void schedule_delayed_file(file_id const & id, file_data const & dat);
261
262 map<file_id, file_data> delayed_files;
263 size_t delayed_writes_size;
264
265 void flush_delayed_writes();
266 void clear_delayed_writes();
267 void write_delayed_file(file_id const & new_id,
268 file_data const & dat);
269
270 void write_delayed_roster(revision_id const & new_id,
271 roster_t const & roster,
272 marking_map const & marking);
273
274 //
275 // --== Reading/writing delta-compressed objects ==--
276 //
277
278 // "do we have any entry for 'ident' that is a base version"
279 bool roster_base_stored(revision_id const & ident);
280 bool roster_base_available(revision_id const & ident);
281
282 // "do we have any entry for 'ident' that is a delta"
283 bool delta_exists(file_id const & ident,
284 file_id const & base,
285 string const & table);
286
287 bool file_or_manifest_base_exists(id const & ident,
288 std::string const & table);
289
290 void get_file_or_manifest_base_unchecked(id const & new_id,
291 data & dat,
292 string const & table);
293 void get_file_or_manifest_delta_unchecked(id const & ident,
294 id const & base,
295 delta & del,
296 string const & table);
297 void get_roster_base(revision_id const & ident,
298 roster_t & roster, marking_map & marking);
299 void get_roster_delta(id const & ident,
300 id const & base,
301 roster_delta & del);
302
303 friend struct file_and_manifest_reconstruction_graph;
304 friend struct roster_reconstruction_graph;
305
306 LRUWritebackCache<id, data, datasz> vcache;
307
308 void get_version(id const & ident,
309 data & dat,
310 string const & data_table,
311 string const & delta_table);
312
313 void drop(id const & base,
314 string const & table);
315 void put_file_delta(file_id const & ident,
316 file_id const & base,
317 file_delta const & del);
318
319 void put_roster_delta(revision_id const & ident,
320 revision_id const & base,
321 roster_delta const & del);
322 void put_version(file_id const & old_id,
323 file_id const & new_id,
324 delta const & del,
325 string const & data_table,
326 string const & delta_table);
327
328 //
329 // --== The ancestry graph ==--
330 //
331 void get_ids(string const & table, set<id> & ids);
332
333 //
334 // --== Rosters ==--
335 //
336 struct extractor;
337 struct file_content_extractor;
338 struct markings_extractor;
339 void extract_from_deltas(revision_id const & id, extractor & x);
340
341 height_map height_cache;
342 parent_id_map parent_cache;
343
344 //
345 // --== Keys ==--
346 //
347 void get_keys(string const & table, vector<rsa_keypair_id> & keys);
348
349 // cache of verifiers for public keys
350 verifier_cache verifiers;
351
352 //
353 // --== Certs ==--
354 //
355 // note: this section is ridiculous. please do something about it.
356 bool cert_exists(cert const & t,
357 string const & table);
358 void put_cert(cert const & t, string const & table);
359 void results_to_certs(results const & res,
360 vector<cert> & certs);
361
362 void get_certs(vector<cert> & certs,
363 string const & table);
364
365 void get_certs(id const & ident,
366 vector<cert> & certs,
367 string const & table);
368
369 void get_certs(cert_name const & name,
370 vector<cert> & certs,
371 string const & table);
372
373 void get_certs(id const & ident,
374 cert_name const & name,
375 vector<cert> & certs,
376 string const & table);
377
378 void get_certs(id const & ident,
379 cert_name const & name,
380 cert_value const & val,
381 vector<cert> & certs,
382 string const & table);
383
384 void get_certs(cert_name const & name,
385 cert_value const & val,
386 vector<cert> & certs,
387 string const & table);
388
389 outdated_indicator_factory cert_stamper;
390
391 void add_prefix_matching_constraint(string const & colname,
392 string const & prefix,
393 query & q);
394};
395
396database_impl::database_impl(system_path const & f) :
397 filename(f),
398 __sql(NULL),
399 transaction_level(0),
400 roster_cache(constants::db_roster_cache_sz,
401 roster_writeback_manager(*this)),
402 delayed_writes_size(0),
403 vcache(constants::db_version_cache_sz)
404{}
405
406database_impl::~database_impl()
407{
408 L(FL("statement cache statistics"));
409 L(FL("prepared %d statements") % statement_cache.size());
410
411 for (map<string, statement>::const_iterator i = statement_cache.begin();
412 i != statement_cache.end(); ++i)
413 L(FL("%d executions of %s") % i->second.count % i->first);
414 // trigger destructors to finalize cached statements
415 statement_cache.clear();
416
417 if (__sql)
418 close();
419}
420
421database::database(app_state & app)
422 : imp(new database_impl(app.opts.dbname)), lua(app.lua)
423{}
424
425database::~database()
426{}
427
428system_path
429database::get_filename()
430{
431 return imp->filename;
432}
433
434bool
435database::is_dbfile(any_path const & file)
436{
437 system_path fn(file); // canonicalize
438 bool same = (imp->filename == fn);
439 if (same)
440 L(FL("'%s' is the database file") % file);
441 return same;
442}
443
444bool
445database::database_specified()
446{
447 return !imp->filename.empty();
448}
449
450void
451database::check_is_not_rosterified()
452{
453 results res;
454 imp->fetch(res, one_col, any_rows,
455 query("SELECT 1 FROM rosters LIMIT 1"));
456 N(res.empty(),
457 F("this database already contains rosters"));
458}
459
460void
461database_impl::check_format()
462{
463 results res;
464
465 // Check for manifests, revisions, rosters, and heights.
466 fetch(res, one_col, any_rows, query("SELECT 1 FROM manifests LIMIT 1"));
467 bool have_manifests = !res.empty();
468 fetch(res, one_col, any_rows, query("SELECT 1 FROM revisions LIMIT 1"));
469 bool have_revisions = !res.empty();
470 fetch(res, one_col, any_rows, query("SELECT 1 FROM rosters LIMIT 1"));
471 bool have_rosters = !res.empty();
472 fetch(res, one_col, any_rows, query("SELECT 1 FROM heights LIMIT 1"));
473 bool have_heights = !res.empty();
474
475
476 if (!have_manifests)
477 {
478 // Must have been changesetified and rosterified already.
479 // Or else the database was just created.
480 // Do we need to regenerate cached data?
481 E(!have_revisions || (have_rosters && have_heights),
482 F("database %s lacks some cached data\n"
483 "run '%s db regenerate_caches' to restore use of this database")
484 % filename % ui.prog_name);
485 }
486 else
487 {
488 // The rosters and heights tables should be empty.
489 I(!have_rosters && !have_heights);
490
491 // they need to either changesetify or rosterify. which?
492 if (have_revisions)
493 E(false,
494 F("database %s contains old-style revisions\n"
495 "if you are a project leader or doing local testing:\n"
496 " see the file UPGRADE for instructions on upgrading.\n"
497 "if you are not a project leader:\n"
498 " wait for a leader to migrate project data, and then\n"
499 " pull into a fresh database.\n"
500 "sorry about the inconvenience.")
501 % filename);
502 else
503 E(false,
504 F("database %s contains manifests but no revisions\n"
505 "this is a very old database; it needs to be upgraded\n"
506 "please see README.changesets for details")
507 % filename);
508 }
509}
510
511static void
512sqlite3_gunzip_fn(sqlite3_context *f, int nargs, sqlite3_value ** args)
513{
514 if (nargs != 1)
515 {
516 sqlite3_result_error(f, "need exactly 1 arg to gunzip()", -1);
517 return;
518 }
519 data unpacked;
520 const char *val = (const char*) sqlite3_value_blob(args[0]);
521 int bytes = sqlite3_value_bytes(args[0]);
522 decode_gzip(gzip<data>(string(val,val+bytes)), unpacked);
523 sqlite3_result_blob(f, unpacked().c_str(), unpacked().size(), SQLITE_TRANSIENT);
524}
525
526struct sqlite3 *
527database_impl::sql(enum open_mode mode)
528{
529 if (! __sql)
530 {
531 check_filename();
532 check_db_exists();
533 open();
534
535 if (mode != schema_bypass_mode)
536 {
537 check_sql_schema(__sql, filename);
538
539 if (mode != format_bypass_mode)
540 check_format();
541 }
542
543 install_functions();
544 }
545 else
546 I(mode == normal_mode);
547
548 return __sql;
549}
550
551void
552database::initialize()
553{
554 imp->check_filename();
555 imp->check_db_nonexistent();
556 imp->open();
557
558 sqlite3 *sql = imp->__sql;
559
560 sqlite3_exec(sql, schema_constant, NULL, NULL, NULL);
561 assert_sqlite3_ok(sql);
562
563 sqlite3_exec(sql, (FL("PRAGMA user_version = %u;")
564 % mtn_creator_code).str().c_str(), NULL, NULL, NULL);
565 assert_sqlite3_ok(sql);
566
567 // make sure what we wanted is what we got
568 check_sql_schema(sql, imp->filename);
569
570 imp->close();
571}
572
573struct
574dump_request
575{
576 dump_request() : sql(), out() {};
577 struct sqlite3 *sql;
578 ostream *out;
579};
580
581static void
582dump_row(ostream &out, sqlite3_stmt *stmt, string const& table_name)
583{
584 out << FL("INSERT INTO %s VALUES(") % table_name;
585 unsigned n = sqlite3_data_count(stmt);
586 for (unsigned i = 0; i < n; ++i)
587 {
588 if (i != 0)
589 out << ',';
590
591 if (sqlite3_column_type(stmt, i) == SQLITE_BLOB)
592 {
593 out << "X'";
594 const char *val = (const char*) sqlite3_column_blob(stmt, i);
595 int bytes = sqlite3_column_bytes(stmt, i);
596 out << encode_hexenc(string(val,val+bytes));
597 out << '\'';
598 }
599 else
600 {
601 const unsigned char *val = sqlite3_column_text(stmt, i);
602 if (val == NULL)
603 out << "NULL";
604 else
605 {
606 out << '\'';
607 for (const unsigned char *cp = val; *cp; ++cp)
608 {
609 if (*cp == '\'')
610 out << "''";
611 else
612 out << *cp;
613 }
614 out << '\'';
615 }
616 }
617 }
618 out << ");\n";
619}
620
621static int
622dump_table_cb(void *data, int n, char **vals, char **cols)
623{
624 dump_request *dump = reinterpret_cast<dump_request *>(data);
625 I(dump != NULL);
626 I(dump->sql != NULL);
627 I(vals != NULL);
628 I(vals[0] != NULL);
629 I(vals[1] != NULL);
630 I(vals[2] != NULL);
631 I(n == 3);
632 I(string(vals[1]) == "table");
633 *(dump->out) << vals[2] << ";\n";
634 string table_name(vals[0]);
635 string query = "SELECT * FROM " + table_name;
636 sqlite3_stmt *stmt = 0;
637 sqlite3_prepare_v2(dump->sql, query.c_str(), -1, &stmt, NULL);
638 assert_sqlite3_ok(dump->sql);
639
640 int stepresult = SQLITE_DONE;
641 do
642 {
643 stepresult = sqlite3_step(stmt);
644 I(stepresult == SQLITE_DONE || stepresult == SQLITE_ROW);
645 if (stepresult == SQLITE_ROW)
646 dump_row(*(dump->out), stmt, table_name);
647 }
648 while (stepresult == SQLITE_ROW);
649
650 sqlite3_finalize(stmt);
651 assert_sqlite3_ok(dump->sql);
652 return 0;
653}
654
655static int
656dump_index_cb(void *data, int n, char **vals, char **cols)
657{
658 dump_request *dump = reinterpret_cast<dump_request *>(data);
659 I(dump != NULL);
660 I(dump->sql != NULL);
661 I(vals != NULL);
662 I(vals[0] != NULL);
663 I(vals[1] != NULL);
664 I(vals[2] != NULL);
665 I(n == 3);
666 I(string(vals[1]) == "index");
667 *(dump->out) << vals[2] << ";\n";
668 return 0;
669}
670
671static int
672dump_user_version_cb(void *data, int n, char **vals, char **cols)
673{
674 dump_request *dump = reinterpret_cast<dump_request *>(data);
675 I(dump != NULL);
676 I(dump->sql != NULL);
677 I(vals != NULL);
678 I(vals[0] != NULL);
679 I(n == 1);
680 *(dump->out) << "PRAGMA user_version = " << vals[0] << ";\n";
681 return 0;
682}
683
684void
685database::dump(ostream & out)
686{
687 ensure_open_for_maintenance();
688
689 {
690 transaction_guard guard(*this);
691 dump_request req;
692 req.out = &out;
693 req.sql = imp->sql();
694 out << "BEGIN EXCLUSIVE;\n";
695 int res;
696 res = sqlite3_exec(req.sql,
697 "SELECT name, type, sql FROM sqlite_master "
698 "WHERE type='table' AND sql NOT NULL "
699 "AND name not like 'sqlite_stat%' "
700 "ORDER BY name",
701 dump_table_cb, &req, NULL);
702 assert_sqlite3_ok(req.sql);
703 res = sqlite3_exec(req.sql,
704 "SELECT name, type, sql FROM sqlite_master "
705 "WHERE type='index' AND sql NOT NULL "
706 "ORDER BY name",
707 dump_index_cb, &req, NULL);
708 assert_sqlite3_ok(req.sql);
709 res = sqlite3_exec(req.sql,
710 "PRAGMA user_version;",
711 dump_user_version_cb, &req, NULL);
712 assert_sqlite3_ok(req.sql);
713 out << "COMMIT;\n";
714 guard.commit();
715 }
716}
717
718void
719database::load(istream & in)
720{
721 string line;
722 string sql_stmt;
723
724 imp->check_filename();
725 imp->check_db_nonexistent();
726 imp->open();
727
728 sqlite3 * sql = imp->__sql;
729
730 // the page size can only be set before any other commands have been executed
731 sqlite3_exec(sql, "PRAGMA page_size=8192", NULL, NULL, NULL);
732 assert_sqlite3_ok(sql);
733
734 while(in)
735 {
736 getline(in, line, ';');
737 sql_stmt += line + ';';
738
739 if (sqlite3_complete(sql_stmt.c_str()))
740 {
741 sqlite3_exec(sql, sql_stmt.c_str(), NULL, NULL, NULL);
742 assert_sqlite3_ok(sql);
743 sql_stmt.clear();
744 }
745 }
746
747 assert_sqlite3_ok(sql);
748}
749
750
751void
752database::debug(string const & sql, ostream & out)
753{
754 ensure_open_for_maintenance();
755
756 results res;
757 imp->fetch(res, any_cols, any_rows, query(sql));
758 out << '\'' << sql << "' -> " << res.size() << " rows\n\n";
759 for (size_t i = 0; i < res.size(); ++i)
760 {
761 for (size_t j = 0; j < res[i].size(); ++j)
762 {
763 if (j != 0)
764 out << " | ";
765 out << res[i][j];
766 }
767 out << '\n';
768 }
769}
770
771// Subroutine of info(). This compares strings that might either be numbers
772// or error messages surrounded by square brackets. We want the longest
773// number, even if there's an error message that's longer than that.
774static bool longest_number(string a, string b)
775{
776 if(a.length() > 0 && a[0] == '[')
777 return true; // b is longer
778 if(b.length() > 0 && b[0] == '[')
779 return false; // a is longer
780
781 return a.length() < b.length();
782}
783
784// Subroutine of info() and some things it calls.
785// Given an informative_failure which is believed to represent an SQLite
786// error, either return a string version of the error message (if it was an
787// SQLite error) or rethrow the execption (if it wasn't).
788static string
789format_sqlite_error_for_info(informative_failure const & e)
790{
791 string err(e.what());
792 string prefix = _("error: ");
793 prefix.append(_("sqlite error: "));
794 if (err.find(prefix) != 0)
795 throw;
796
797 err.replace(0, prefix.length(), "[");
798 string::size_type nl = err.find('\n');
799 if (nl != string::npos)
800 err.erase(nl);
801
802 err.append("]");
803 return err;
804}
805
806// Subroutine of info(). Pretty-print the database's "creator code", which
807// is a 32-bit unsigned number that we interpret as a four-character ASCII
808// string, provided that all four characters are graphic. (On disk, it's
809// stored in the "user version" field of the database.)
810static string
811format_creator_code(u32 code)
812{
813 char buf[5];
814 string result;
815
816 if (code == 0)
817 return _("not set");
818
819 buf[4] = '\0';
820 buf[3] = ((code & 0x000000ff) >> 0);
821 buf[2] = ((code & 0x0000ff00) >> 8);
822 buf[1] = ((code & 0x00ff0000) >> 16);
823 buf[0] = ((code & 0xff000000) >> 24);
824
825 if (isgraph(buf[0]) && isgraph(buf[1]) && isgraph(buf[2]) && isgraph(buf[3]))
826 result = (FL("%s (0x%08x)") % buf % code).str();
827 else
828 result = (FL("0x%08x") % code).str();
829 if (code != mtn_creator_code)
830 result += _(" (not a monotone database)");
831 return result;
832}
833
834
835void
836database::info(ostream & out)
837{
838 // don't check the schema
839 ensure_open_for_maintenance();
840
841 // do a dummy query to confirm that the database file is an sqlite3
842 // database. (this doesn't happen on open() because sqlite postpones the
843 // actual file open until the first access. we can't piggyback it on the
844 // query of the user version because there's a bug in sqlite 3.3.10:
845 // the routine that reads meta-values from the database header does not
846 // check the file format. reported as sqlite bug #2182.)
847 sqlite3_exec(imp->__sql, "SELECT 1 FROM sqlite_master LIMIT 0", 0, 0, 0);
848 assert_sqlite3_ok(imp->__sql);
849
850 u32 ccode;
851 {
852 results res;
853 imp->fetch(res, one_col, one_row, query("PRAGMA user_version"));
854 I(res.size() == 1);
855 ccode = lexical_cast<u32>(res[0][0]);
856 }
857
858 vector<string> counts;
859 counts.push_back(imp->count("rosters"));
860 counts.push_back(imp->count("roster_deltas"));
861 counts.push_back(imp->count("files"));
862 counts.push_back(imp->count("file_deltas"));
863 counts.push_back(imp->count("revisions"));
864 counts.push_back(imp->count("revision_ancestry"));
865 counts.push_back(imp->count("revision_certs"));
866
867 {
868 results res;
869 try
870 {
871 imp->fetch(res, one_col, any_rows,
872 query("SELECT node FROM next_roster_node_number"));
873 if (res.empty())
874 counts.push_back("0");
875 else
876 {
877 I(res.size() == 1);
878 u64 n = lexical_cast<u64>(res[0][0]) - 1;
879 counts.push_back((F("%u") % n).str());
880 }
881 }
882 catch (informative_failure const & e)
883 {
884 counts.push_back(format_sqlite_error_for_info(e));
885 }
886 }
887
888 vector<string> bytes;
889 {
890 u64 total = 0;
891 bytes.push_back(imp->space("rosters",
892 "length(id) + length(checksum) + length(data)",
893 total));
894 bytes.push_back(imp->space("roster_deltas",
895 "length(id) + length(checksum)"
896 "+ length(base) + length(delta)", total));
897 bytes.push_back(imp->space("files", "length(id) + length(data)", total));
898 bytes.push_back(imp->space("file_deltas",
899 "length(id) + length(base) + length(delta)", total));
900 bytes.push_back(imp->space("revisions", "length(id) + length(data)", total));
901 bytes.push_back(imp->space("revision_ancestry",
902 "length(parent) + length(child)", total));
903 bytes.push_back(imp->space("revision_certs",
904 "length(hash) + length(id) + length(name)"
905 "+ length(value) + length(keypair)"
906 "+ length(signature)", total));
907 bytes.push_back(imp->space("heights", "length(revision) + length(height)",
908 total));
909 bytes.push_back((F("%u") % total).str());
910 }
911
912 // pad each vector's strings on the left with spaces to make them all the
913 // same length
914 {
915 string::size_type width
916 = max_element(counts.begin(), counts.end(), longest_number)->length();
917 for(vector<string>::iterator i = counts.begin(); i != counts.end(); i++)
918 if (width > i->length() && (*i)[0] != '[')
919 i->insert(0, width - i->length(), ' ');
920
921 width = max_element(bytes.begin(), bytes.end(), longest_number)->length();
922 for(vector<string>::iterator i = bytes.begin(); i != bytes.end(); i++)
923 if (width > i->length() && (*i)[0] != '[')
924 i->insert(0, width - i->length(), ' ');
925 }
926
927 i18n_format form =
928 F("creator code : %s\n"
929 "schema version : %s\n"
930 "counts:\n"
931 " full rosters : %s\n"
932 " roster deltas : %s\n"
933 " full files : %s\n"
934 " file deltas : %s\n"
935 " revisions : %s\n"
936 " ancestry edges : %s\n"
937 " certs : %s\n"
938 " logical files : %s\n"
939 "bytes:\n"
940 " full rosters : %s\n"
941 " roster deltas : %s\n"
942 " full files : %s\n"
943 " file deltas : %s\n"
944 " revisions : %s\n"
945 " cached ancestry : %s\n"
946 " certs : %s\n"
947 " heights : %s\n"
948 " total : %s\n"
949 "database:\n"
950 " page size : %s\n"
951 " cache size : %s"
952 );
953
954 form = form % format_creator_code(ccode);
955 form = form % describe_sql_schema(imp->__sql);
956
957 for (vector<string>::iterator i = counts.begin(); i != counts.end(); i++)
958 form = form % *i;
959
960 for (vector<string>::iterator i = bytes.begin(); i != bytes.end(); i++)
961 form = form % *i;
962
963 form = form % imp->page_size();
964 form = form % imp->cache_size();
965
966 out << form.str() << '\n'; // final newline is kept out of the translation
967}
968
969void
970database::version(ostream & out)
971{
972 ensure_open_for_maintenance();
973 out << (F("database schema version: %s")
974 % describe_sql_schema(imp->__sql)).str()
975 << '\n';
976}
977
978void
979database::migrate(key_store & keys)
980{
981 ensure_open_for_maintenance();
982 migrate_sql_schema(imp->__sql, keys, get_filename());
983}
984
985void
986database::test_migration_step(key_store & keys, string const & schema)
987{
988 ensure_open_for_maintenance();
989 ::test_migration_step(imp->__sql, keys, get_filename(), schema);
990}
991
992void
993database::ensure_open()
994{
995 imp->sql();
996}
997
998void
999database::ensure_open_for_format_changes()
1000{
1001 imp->sql(format_bypass_mode);
1002}
1003
1004void
1005database::ensure_open_for_maintenance()
1006{
1007 imp->sql(schema_bypass_mode);
1008}
1009
1010void
1011database_impl::execute(query const & query)
1012{
1013 results res;
1014 fetch(res, 0, 0, query);
1015}
1016
1017void
1018database_impl::fetch(results & res,
1019 int const want_cols,
1020 int const want_rows,
1021 query const & query)
1022{
1023 int nrow;
1024 int ncol;
1025 int rescode;
1026
1027 res.clear();
1028 res.resize(0);
1029
1030 map<string, statement>::iterator i = statement_cache.find(query.sql_cmd);
1031 if (i == statement_cache.end())
1032 {
1033 statement_cache.insert(make_pair(query.sql_cmd, statement()));
1034 i = statement_cache.find(query.sql_cmd);
1035 I(i != statement_cache.end());
1036
1037 const char * tail;
1038 sqlite3_prepare_v2(sql(), query.sql_cmd.c_str(), -1, i->second.stmt.paddr(), &tail);
1039 assert_sqlite3_ok(sql());
1040 L(FL("prepared statement %s") % query.sql_cmd);
1041
1042 // no support for multiple statements here
1043 E(*tail == 0,
1044 F("multiple statements in query: %s") % query.sql_cmd);
1045 }
1046
1047 ncol = sqlite3_column_count(i->second.stmt());
1048
1049 E(want_cols == any_cols || want_cols == ncol,
1050 F("wanted %d columns got %d in query: %s") % want_cols % ncol % query.sql_cmd);
1051
1052 // bind parameters for this execution
1053
1054 int params = sqlite3_bind_parameter_count(i->second.stmt());
1055
1056 // Ensure that exactly the right number of parameters were given
1057 I(params == int(query.args.size()));
1058
1059 L(FL("binding %d parameters for %s") % params % query.sql_cmd);
1060
1061 for (int param = 1; param <= params; param++)
1062 {
1063 // profiling finds this logging to be quite expensive
1064 if (global_sanity.debug_p())
1065 {
1066 string prefix;
1067 string log(query.args[param-1].data);
1068
1069 if (query.args[param-1].type == query_param::blob)
1070 {
1071 prefix = "x";
1072 log = encode_hexenc(log);
1073 }
1074
1075 if (log.size() > constants::db_log_line_sz)
1076 log = log.substr(0, constants::db_log_line_sz - 2) + "..";
1077
1078 L(FL("binding %d with value '%s'") % param % log);
1079 }
1080
1081 switch (idx(query.args, param - 1).type)
1082 {
1083 case query_param::text:
1084 sqlite3_bind_text(i->second.stmt(), param,
1085 idx(query.args, param - 1).data.c_str(), -1,
1086 SQLITE_STATIC);
1087 break;
1088 case query_param::blob:
1089 {
1090 string const & data = idx(query.args, param - 1).data;
1091 sqlite3_bind_blob(i->second.stmt(), param,
1092 data.data(), data.size(),
1093 SQLITE_STATIC);
1094 }
1095 break;
1096 default:
1097 I(false);
1098 }
1099
1100 assert_sqlite3_ok(sql());
1101 }
1102
1103 // execute and process results
1104
1105 nrow = 0;
1106 for (rescode = sqlite3_step(i->second.stmt()); rescode == SQLITE_ROW;
1107 rescode = sqlite3_step(i->second.stmt()))
1108 {
1109 vector<string> row;
1110 for (int col = 0; col < ncol; col++)
1111 {
1112 const char * value = (const char*)sqlite3_column_blob(i->second.stmt(), col);
1113 int bytes = sqlite3_column_bytes(i->second.stmt(), col);
1114 E(value, F("null result in query: %s") % query.sql_cmd);
1115 row.push_back(string(value, value + bytes));
1116 //L(FL("row %d col %d value='%s'") % nrow % col % value);
1117 }
1118 res.push_back(row);
1119 }
1120
1121 if (rescode != SQLITE_DONE)
1122 assert_sqlite3_ok(sql());
1123
1124 sqlite3_reset(i->second.stmt());
1125 assert_sqlite3_ok(sql());
1126
1127 nrow = res.size();
1128
1129 i->second.count++;
1130
1131 E(want_rows == any_rows || want_rows == nrow,
1132 F("wanted %d rows got %d in query: %s") % want_rows % nrow % query.sql_cmd);
1133}
1134
1135bool
1136database_impl::table_has_entry(id const & key,
1137 std::string const & column,
1138 std::string const & table)
1139{
1140 results res;
1141 query q("SELECT 1 FROM " + table + " WHERE " + column + " = ? LIMIT 1");
1142 fetch(res, one_col, any_rows, q % blob(key()));
1143 return !res.empty();
1144}
1145
1146// general application-level logic
1147
1148void
1149database_impl::begin_transaction(bool exclusive)
1150{
1151 if (transaction_level == 0)
1152 {
1153 I(delayed_files.empty());
1154 I(roster_cache.all_clean());
1155 if (exclusive)
1156 execute(query("BEGIN EXCLUSIVE"));
1157 else
1158 execute(query("BEGIN DEFERRED"));
1159 transaction_exclusive = exclusive;
1160 }
1161 else
1162 {
1163 // You can't start an exclusive transaction within a non-exclusive
1164 // transaction
1165 I(!exclusive || transaction_exclusive);
1166 }
1167 transaction_level++;
1168}
1169
1170
1171static size_t
1172size_delayed_file(file_id const & id, file_data const & dat)
1173{
1174 return id.inner()().size() + dat.inner()().size();
1175}
1176
1177bool
1178database_impl::have_delayed_file(file_id const & id)
1179{
1180 return delayed_files.find(id) != delayed_files.end();
1181}
1182
1183void
1184database_impl::load_delayed_file(file_id const & id, file_data & dat)
1185{
1186 dat = safe_get(delayed_files, id);
1187}
1188
1189// precondition: have_delayed_file(an_id) == true
1190void
1191database_impl::cancel_delayed_file(file_id const & an_id)
1192{
1193 file_data const & dat = safe_get(delayed_files, an_id);
1194 size_t cancel_size = size_delayed_file(an_id, dat);
1195 I(cancel_size <= delayed_writes_size);
1196 delayed_writes_size -= cancel_size;
1197
1198 safe_erase(delayed_files, an_id);
1199}
1200
1201void
1202database_impl::drop_or_cancel_file(file_id const & id)
1203{
1204 if (have_delayed_file(id))
1205 cancel_delayed_file(id);
1206 else
1207 drop(id.inner(), "files");
1208}
1209
1210void
1211database_impl::schedule_delayed_file(file_id const & an_id,
1212 file_data const & dat)
1213{
1214 if (!have_delayed_file(an_id))
1215 {
1216 safe_insert(delayed_files, make_pair(an_id, dat));
1217 delayed_writes_size += size_delayed_file(an_id, dat);
1218 }
1219 if (delayed_writes_size > constants::db_max_delayed_file_bytes)
1220 flush_delayed_writes();
1221}
1222
1223void
1224database_impl::flush_delayed_writes()
1225{
1226 for (map<file_id, file_data>::const_iterator i = delayed_files.begin();
1227 i != delayed_files.end(); ++i)
1228 write_delayed_file(i->first, i->second);
1229 clear_delayed_writes();
1230}
1231
1232void
1233database_impl::clear_delayed_writes()
1234{
1235 delayed_files.clear();
1236 delayed_writes_size = 0;
1237}
1238
1239void
1240database_impl::roster_writeback_manager::writeout(revision_id const & id,
1241 cached_roster const & cr)
1242{
1243 I(cr.first);
1244 I(cr.second);
1245 imp.write_delayed_roster(id, *(cr.first), *(cr.second));
1246}
1247
1248void
1249database_impl::commit_transaction()
1250{
1251 if (transaction_level == 1)
1252 {
1253 flush_delayed_writes();
1254 roster_cache.clean_all();
1255 execute(query("COMMIT"));
1256 }
1257 transaction_level--;
1258}
1259
1260void
1261database_impl::rollback_transaction()
1262{
1263 if (transaction_level == 1)
1264 {
1265 clear_delayed_writes();
1266 roster_cache.clear_and_drop_writes();
1267 execute(query("ROLLBACK"));
1268 }
1269 transaction_level--;
1270}
1271
1272
1273bool
1274database_impl::file_or_manifest_base_exists(id const & ident,
1275 string const & table)
1276{
1277 // just check for a delayed file, since there are no delayed manifests
1278 if (have_delayed_file(file_id(ident)))
1279 return true;
1280 return table_has_entry(ident, "id", table);
1281}
1282
1283bool
1284database::file_or_manifest_base_exists(file_id const & ident,
1285 string const & table)
1286{
1287 return imp->file_or_manifest_base_exists(ident.inner(), table);
1288}
1289
1290// returns true if we are currently storing (or planning to store) a
1291// full-text for 'ident'
1292bool
1293database_impl::roster_base_stored(revision_id const & ident)
1294{
1295 if (roster_cache.exists(ident) && roster_cache.is_dirty(ident))
1296 return true;
1297 return table_has_entry(ident.inner(), "id", "rosters");
1298}
1299
1300// returns true if we currently have a full-text for 'ident' available
1301// (possibly cached). Warning: the results of this method are invalidated
1302// by calling roster_cache.insert_{clean,dirty}, because they can trigger
1303// cache cleaning.
1304bool
1305database_impl::roster_base_available(revision_id const & ident)
1306{
1307 if (roster_cache.exists(ident))
1308 return true;
1309 return table_has_entry(ident.inner(), "id", "rosters");
1310}
1311
1312bool
1313database::delta_exists(id const & ident,
1314 string const & table)
1315{
1316 return imp->table_has_entry(ident, "id", table);
1317}
1318
1319bool
1320database_impl::delta_exists(file_id const & ident,
1321 file_id const & base,
1322 string const & table)
1323{
1324 results res;
1325 query q("SELECT 1 FROM " + table + " WHERE id = ? and base = ? LIMIT 1");
1326 fetch(res, one_col, any_rows,
1327 q % blob(ident.inner()()) % blob(base.inner()()));
1328 return !res.empty();
1329}
1330
1331string
1332database_impl::count(string const & table)
1333{
1334 try
1335 {
1336 results res;
1337 query q("SELECT COUNT(*) FROM " + table);
1338 fetch(res, one_col, one_row, q);
1339 return (F("%u") % lexical_cast<u64>(res[0][0])).str();
1340 }
1341 catch (informative_failure const & e)
1342 {
1343 return format_sqlite_error_for_info(e);
1344 }
1345
1346}
1347
1348string
1349database_impl::space(string const & table, string const & rowspace, u64 & total)
1350{
1351 try
1352 {
1353 results res;
1354 // SUM({empty set}) is NULL; TOTAL({empty set}) is 0.0
1355 query q("SELECT TOTAL(" + rowspace + ") FROM " + table);
1356 fetch(res, one_col, one_row, q);
1357 u64 bytes = static_cast<u64>(lexical_cast<double>(res[0][0]));
1358 total += bytes;
1359 return (F("%u") % bytes).str();
1360 }
1361 catch (informative_failure & e)
1362 {
1363 return format_sqlite_error_for_info(e);
1364 }
1365}
1366
1367unsigned int
1368database_impl::page_size()
1369{
1370 results res;
1371 query q("PRAGMA page_size");
1372 fetch(res, one_col, one_row, q);
1373 return lexical_cast<unsigned int>(res[0][0]);
1374}
1375
1376unsigned int
1377database_impl::cache_size()
1378{
1379 // This returns the persistent (default) cache size. It's possible to
1380 // override this setting transiently at runtime by setting PRAGMA
1381 // cache_size.
1382 results res;
1383 query q("PRAGMA default_cache_size");
1384 fetch(res, one_col, one_row, q);
1385 return lexical_cast<unsigned int>(res[0][0]);
1386}
1387
1388void
1389database_impl::get_ids(string const & table, set<id> & ids)
1390{
1391 results res;
1392 query q("SELECT id FROM " + table);
1393 fetch(res, one_col, any_rows, q);
1394
1395 for (size_t i = 0; i < res.size(); ++i)
1396 {
1397 ids.insert(id(res[i][0]));
1398 }
1399}
1400
1401// for files and legacy manifest support
1402void
1403database_impl::get_file_or_manifest_base_unchecked(id const & ident,
1404 data & dat,
1405 string const & table)
1406{
1407 if (have_delayed_file(file_id(ident)))
1408 {
1409 file_data tmp;
1410 load_delayed_file(file_id(ident), tmp);
1411 dat = tmp.inner();
1412 return;
1413 }
1414
1415 results res;
1416 query q("SELECT data FROM " + table + " WHERE id = ?");
1417 fetch(res, one_col, one_row, q % blob(ident()));
1418
1419 gzip<data> rdata(res[0][0]);
1420 data rdata_unpacked;
1421 decode_gzip(rdata,rdata_unpacked);
1422
1423 dat = rdata_unpacked;
1424}
1425
1426// for files and legacy manifest support
1427void
1428database_impl::get_file_or_manifest_delta_unchecked(id const & ident,
1429 id const & base,
1430 delta & del,
1431 string const & table)
1432{
1433 I(ident() != "");
1434 I(base() != "");
1435 results res;
1436 query q("SELECT delta FROM " + table + " WHERE id = ? AND base = ?");
1437 fetch(res, one_col, one_row,
1438 q % blob(ident()) % blob(base()));
1439
1440 gzip<delta> del_packed(res[0][0]);
1441 decode_gzip(del_packed, del);
1442}
1443
1444void
1445database_impl::get_roster_base(revision_id const & ident,
1446 roster_t & roster, marking_map & marking)
1447{
1448 if (roster_cache.exists(ident))
1449 {
1450 cached_roster cr;
1451 roster_cache.fetch(ident, cr);
1452 I(cr.first);
1453 roster = *(cr.first);
1454 I(cr.second);
1455 marking = *(cr.second);
1456 return;
1457 }
1458 results res;
1459 query q("SELECT checksum, data FROM rosters WHERE id = ?");
1460 fetch(res, 2, one_row, q % blob(ident.inner()()));
1461
1462 id checksum(res[0][0]);
1463 id calculated;
1464 calculate_ident(data(res[0][1]), calculated);
1465 I(calculated == checksum);
1466
1467 gzip<data> dat_packed(res[0][1]);
1468 data dat;
1469 decode_gzip(dat_packed, dat);
1470 read_roster_and_marking(roster_data(dat), roster, marking);
1471}
1472
1473void
1474database_impl::get_roster_delta(id const & ident,
1475 id const & base,
1476 roster<delta> & del)
1477{
1478 results res;
1479 query q("SELECT checksum, delta FROM roster_deltas WHERE id = ? AND base = ?");
1480 fetch(res, 2, one_row, q % blob(ident()) % blob(base()));
1481
1482 id checksum(res[0][0]);
1483 id calculated;
1484 calculate_ident(data(res[0][1]), calculated);
1485 I(calculated == checksum);
1486
1487 gzip<delta> del_packed(res[0][1]);
1488 delta tmp;
1489 decode_gzip(del_packed, tmp);
1490 del = roster<delta>(tmp);
1491}
1492
1493void
1494database_impl::write_delayed_file(file_id const & ident,
1495 file_data const & dat)
1496{
1497 gzip<data> dat_packed;
1498 encode_gzip(dat.inner(), dat_packed);
1499
1500 // ident is a hash, which we should check
1501 I(!null_id(ident));
1502 file_id tid;
1503 calculate_ident(dat, tid);
1504 MM(ident);
1505 MM(tid);
1506 I(tid == ident);
1507 // and then write things to the db
1508 query q("INSERT INTO files (id, data) VALUES (?, ?)");
1509 execute(q % blob(ident.inner()()) % blob(dat_packed()));
1510}
1511
1512void
1513database_impl::write_delayed_roster(revision_id const & ident,
1514 roster_t const & roster,
1515 marking_map const & marking)
1516{
1517 roster_data dat;
1518 write_roster_and_marking(roster, marking, dat);
1519 gzip<data> dat_packed;
1520 encode_gzip(dat.inner(), dat_packed);
1521
1522 // ident is a number, and we should calculate a checksum on what
1523 // we write
1524 id checksum;
1525 calculate_ident(data(dat_packed()), checksum);
1526
1527 // and then write it
1528 execute(query("INSERT INTO rosters (id, checksum, data) VALUES (?, ?, ?)")
1529 % blob(ident.inner()())
1530 % blob(checksum())
1531 % blob(dat_packed()));
1532}
1533
1534
1535void
1536database::put_file_delta(file_id const & ident,
1537 file_id const & base,
1538 file_delta const & del)
1539{
1540 // nb: delta schema is (id, base, delta)
1541 I(!null_id(ident));
1542 I(!null_id(base));
1543
1544 gzip<delta> del_packed;
1545 encode_gzip(del.inner(), del_packed);
1546
1547 imp->execute(query("INSERT INTO file_deltas VALUES (?, ?, ?)")
1548 % blob(ident.inner()())
1549 % blob(base.inner()())
1550 % blob(del_packed()));
1551}
1552
1553void
1554database_impl::put_roster_delta(revision_id const & ident,
1555 revision_id const & base,
1556 roster_delta const & del)
1557{
1558 gzip<delta> del_packed;
1559 encode_gzip(del.inner(), del_packed);
1560
1561 id checksum;
1562 calculate_ident(data(del_packed()), checksum);
1563
1564 query q("INSERT INTO roster_deltas (id, base, checksum, delta) VALUES (?, ?, ?, ?)");
1565 execute(q
1566 % blob(ident.inner()())
1567 % blob(base.inner()())
1568 % blob(checksum())
1569 % blob(del_packed()));
1570}
1571
1572struct file_and_manifest_reconstruction_graph : public reconstruction_graph
1573{
1574 database_impl & imp;
1575 string const & data_table;
1576 string const & delta_table;
1577
1578 file_and_manifest_reconstruction_graph(database_impl & imp,
1579 string const & data_table,
1580 string const & delta_table)
1581 : imp(imp), data_table(data_table), delta_table(delta_table)
1582 {}
1583 virtual bool is_base(id const & node) const
1584 {
1585 return imp.vcache.exists(node)
1586 || imp.file_or_manifest_base_exists(node, data_table);
1587 }
1588 virtual void get_next(id const & from, set<id> & next) const
1589 {
1590 next.clear();
1591 results res;
1592 query q("SELECT base FROM " + delta_table + " WHERE id = ?");
1593 imp.fetch(res, one_col, any_rows, q % blob(from()));
1594 for (results::const_iterator i = res.begin(); i != res.end(); ++i)
1595 next.insert(id((*i)[0]));
1596 }
1597};
1598
1599// used for files and legacy manifest migration
1600void
1601database_impl::get_version(id const & ident,
1602 data & dat,
1603 string const & data_table,
1604 string const & delta_table)
1605{
1606 I(ident() != "");
1607
1608 reconstruction_path selected_path;
1609 {
1610 file_and_manifest_reconstruction_graph graph(*this, data_table, delta_table);
1611 get_reconstruction_path(ident, graph, selected_path);
1612 }
1613
1614 I(!selected_path.empty());
1615
1616 id curr = selected_path.back();
1617 selected_path.pop_back();
1618 data begin;
1619
1620 if (vcache.exists(curr))
1621 I(vcache.fetch(curr, begin));
1622 else
1623 get_file_or_manifest_base_unchecked(curr, begin, data_table);
1624
1625 shared_ptr<delta_applicator> appl = new_piecewise_applicator();
1626 appl->begin(begin());
1627
1628 for (reconstruction_path::reverse_iterator i = selected_path.rbegin();
1629 i != selected_path.rend(); ++i)
1630 {
1631 id const nxt = id(*i);
1632
1633 if (!vcache.exists(curr))
1634 {
1635 string tmp;
1636 appl->finish(tmp);
1637 vcache.insert_clean(curr, data(tmp));
1638 }
1639
1640 if (global_sanity.debug_p())
1641 L(FL("following delta %s -> %s") % curr % nxt);
1642 delta del;
1643 get_file_or_manifest_delta_unchecked(nxt, curr, del, delta_table);
1644 apply_delta(appl, del());
1645
1646 appl->next();
1647 curr = nxt;
1648 }
1649
1650 string tmp;
1651 appl->finish(tmp);
1652 dat = data(tmp);
1653
1654 id final;
1655 calculate_ident(dat, final);
1656 I(final == ident);
1657
1658 if (!vcache.exists(ident))
1659 vcache.insert_clean(ident, dat);
1660}
1661
1662struct roster_reconstruction_graph : public reconstruction_graph
1663{
1664 database_impl & imp;
1665 roster_reconstruction_graph(database_impl & imp) : imp(imp) {}
1666 virtual bool is_base(id const & node) const
1667 {
1668 return imp.roster_base_available(revision_id(node));
1669 }
1670 virtual void get_next(id const & from, set<id> & next) const
1671 {
1672 next.clear();
1673 results res;
1674 query q("SELECT base FROM roster_deltas WHERE id = ?");
1675 imp.fetch(res, one_col, any_rows, q % blob(from()));
1676 for (results::const_iterator i = res.begin(); i != res.end(); ++i)
1677 next.insert(id((*i)[0]));
1678 }
1679};
1680
1681struct database_impl::extractor
1682{
1683 virtual bool look_at_delta(roster_delta const & del) = 0;
1684 virtual void look_at_roster(roster_t const & roster, marking_map const & mm) = 0;
1685 virtual ~extractor() {};
1686};
1687
1688struct database_impl::markings_extractor : public database_impl::extractor
1689{
1690private:
1691 node_id const & nid;
1692 marking_t & markings;
1693
1694public:
1695 markings_extractor(node_id const & _nid, marking_t & _markings) :
1696 nid(_nid), markings(_markings) {} ;
1697
1698 bool look_at_delta(roster_delta const & del)
1699 {
1700 return try_get_markings_from_roster_delta(del, nid, markings);
1701 }
1702
1703 void look_at_roster(roster_t const & roster, marking_map const & mm)
1704 {
1705 marking_map::const_iterator mmi =
1706 mm.find(nid);
1707 I(mmi != mm.end());
1708 markings = mmi->second;
1709 }
1710};
1711
1712struct database_impl::file_content_extractor : database_impl::extractor
1713{
1714private:
1715 node_id const & nid;
1716 file_id & content;
1717
1718public:
1719 file_content_extractor(node_id const & _nid, file_id & _content) :
1720 nid(_nid), content(_content) {} ;
1721
1722 bool look_at_delta(roster_delta const & del)
1723 {
1724 return try_get_content_from_roster_delta(del, nid, content);
1725 }
1726
1727 void look_at_roster(roster_t const & roster, marking_map const & mm)
1728 {
1729 if (roster.has_node(nid))
1730 content = downcast_to_file_t(roster.get_node(nid))->content;
1731 else
1732 content = file_id();
1733 }
1734};
1735
1736void
1737database_impl::extract_from_deltas(revision_id const & ident, extractor & x)
1738{
1739 reconstruction_path selected_path;
1740 {
1741 roster_reconstruction_graph graph(*this);
1742 {
1743 // we look at the nearest delta(s) first, without constructing the
1744 // whole path, as that would be a rather expensive operation.
1745 //
1746 // the reason why this strategy is worth the effort is, that in most
1747 // cases we are looking at the parent of a (content-)marked node, thus
1748 // the information we are for is right there in the delta leading to
1749 // this node.
1750 //
1751 // recording the deltas visited here in a set as to avoid inspecting
1752 // them later seems to be of little value, as it imposes a cost here,
1753 // but can seldom be exploited.
1754 set<id> deltas;
1755 graph.get_next(ident.inner(), deltas);
1756 for (set<id>::const_iterator i = deltas.begin();
1757 i != deltas.end(); ++i)
1758 {
1759 roster_delta del;
1760 get_roster_delta(ident.inner(), *i, del);
1761 bool found = x.look_at_delta(del);
1762 if (found)
1763 return;
1764 }
1765 }
1766 get_reconstruction_path(ident.inner(), graph, selected_path);
1767 }
1768
1769 int path_length(selected_path.size());
1770 int i(0);
1771 id target_rev;
1772
1773 for (reconstruction_path::const_iterator p = selected_path.begin();
1774 p != selected_path.end(); ++p)
1775 {
1776 if (i > 0)
1777 {
1778 roster_delta del;
1779 get_roster_delta(target_rev, id(*p), del);
1780 bool found = x.look_at_delta(del);
1781 if (found)
1782 return;
1783 }
1784 if (i == path_length-1)
1785 {
1786 // last iteration, we have reached a roster base
1787 roster_t roster;
1788 marking_map mm;
1789 get_roster_base(revision_id(*p), roster, mm);
1790 x.look_at_roster(roster, mm);
1791 return;
1792 }
1793 target_rev = id(*p);
1794 ++i;
1795 }
1796}
1797
1798void
1799database::get_markings(revision_id const & id,
1800 node_id const & nid,
1801 marking_t & markings)
1802{
1803 database_impl::markings_extractor x(nid, markings);
1804 imp->extract_from_deltas(id, x);
1805}
1806
1807void
1808database::get_file_content(revision_id const & id,
1809 node_id const & nid,
1810 file_id & content)
1811{
1812 // the imaginary root revision doesn't have any file.
1813 if (null_id(id))
1814 {
1815 content = file_id();
1816 return;
1817 }
1818 database_impl::file_content_extractor x(nid, content);
1819 imp->extract_from_deltas(id, x);
1820}
1821
1822void
1823database::get_roster_version(revision_id const & ros_id,
1824 cached_roster & cr)
1825{
1826 // if we already have it, exit early
1827 if (imp->roster_cache.exists(ros_id))
1828 {
1829 imp->roster_cache.fetch(ros_id, cr);
1830 return;
1831 }
1832
1833 reconstruction_path selected_path;
1834 {
1835 roster_reconstruction_graph graph(*imp);
1836 get_reconstruction_path(ros_id.inner(), graph, selected_path);
1837 }
1838
1839 id curr(selected_path.back());
1840 selected_path.pop_back();
1841 // we know that this isn't already in the cache (because of the early exit
1842 // above), so we should create new objects and spend time filling them in.
1843 shared_ptr<roster_t> roster(new roster_t);
1844 shared_ptr<marking_map> marking(new marking_map);
1845 imp->get_roster_base(revision_id(curr), *roster, *marking);
1846
1847 for (reconstruction_path::reverse_iterator i = selected_path.rbegin();
1848 i != selected_path.rend(); ++i)
1849 {
1850 id const nxt(*i);
1851 if (global_sanity.debug_p())
1852 L(FL("following delta %s -> %s") % curr % nxt);
1853 roster_delta del;
1854 imp->get_roster_delta(nxt, curr, del);
1855 apply_roster_delta(del, *roster, *marking);
1856 curr = nxt;
1857 }
1858
1859 // Double-check that the thing we got out looks okay. We know that when
1860 // the roster was written to the database, it passed both of these tests,
1861 // and we also know that the data on disk has passed our checks for data
1862 // corruption -- so in theory, we know that what we got out is exactly
1863 // what we put in, and these checks are redundant. (They cannot catch all
1864 // possible errors in any case, e.g., they don't test that the marking is
1865 // correct.) What they can do, though, is serve as a sanity check on the
1866 // delta reconstruction code; if there is a bug where we put something
1867 // into the database and then later get something different back out, then
1868 // this is the only thing that can catch it.
1869 roster->check_sane_against(*marking);
1870 manifest_id expected_mid, actual_mid;
1871 get_revision_manifest(ros_id, expected_mid);
1872 calculate_ident(*roster, actual_mid);
1873 I(expected_mid == actual_mid);
1874
1875 // const'ify the objects, to save them and pass them out
1876 cr.first = roster;
1877 cr.second = marking;
1878 imp->roster_cache.insert_clean(ros_id, cr);
1879}
1880
1881
1882void
1883database_impl::drop(id const & ident,
1884 string const & table)
1885{
1886 string drop = "DELETE FROM " + table + " WHERE id = ?";
1887 execute(query(drop) % blob(ident()));
1888}
1889
1890// ------------------------------------------------------------
1891// -- --
1892// -- public interface follows --
1893// -- --
1894// ------------------------------------------------------------
1895
1896bool
1897database::file_version_exists(file_id const & id)
1898{
1899 return delta_exists(id.inner(), "file_deltas")
1900 || imp->file_or_manifest_base_exists(id.inner(), "files");
1901}
1902
1903bool
1904database::roster_version_exists(revision_id const & id)
1905{
1906 return delta_exists(id.inner(), "roster_deltas")
1907 || imp->roster_base_available(id);
1908}
1909
1910bool
1911database::revision_exists(revision_id const & id)
1912{
1913 results res;
1914 query q("SELECT id FROM revisions WHERE id = ?");
1915 imp->fetch(res, one_col, any_rows, q % blob(id.inner()()));
1916 I(res.size() <= 1);
1917 return res.size() == 1;
1918}
1919
1920void
1921database::get_file_ids(set<file_id> & ids)
1922{
1923 ids.clear();
1924 set<id> tmp;
1925 imp->get_ids("files", tmp);
1926 imp->get_ids("file_deltas", tmp);
1927 add_decoration_to_container(tmp, ids);
1928}
1929
1930void
1931database::get_revision_ids(set<revision_id> & ids)
1932{
1933 ids.clear();
1934 set<id> tmp;
1935 imp->get_ids("revisions", tmp);
1936 add_decoration_to_container(tmp, ids);
1937}
1938
1939void
1940database::get_roster_ids(set<revision_id> & ids)
1941{
1942 ids.clear();
1943 set<id> tmp;
1944 imp->get_ids("rosters", tmp);
1945 add_decoration_to_container(tmp, ids);
1946 imp->get_ids("roster_deltas", tmp);
1947 add_decoration_to_container(tmp, ids);
1948}
1949
1950void
1951database::get_file_version(file_id const & id,
1952 file_data & dat)
1953{
1954 data tmp;
1955 imp->get_version(id.inner(), tmp, "files", "file_deltas");
1956 dat = file_data(tmp);
1957}
1958
1959void
1960database::get_manifest_version(manifest_id const & id,
1961 manifest_data & dat)
1962{
1963 data tmp;
1964 imp->get_version(id.inner(), tmp, "manifests", "manifest_deltas");
1965 dat = manifest_data(tmp);
1966}
1967
1968void
1969database::put_file(file_id const & id,
1970 file_data const & dat)
1971{
1972 if (file_version_exists(id))
1973 L(FL("file version '%s' already exists in db") % id);
1974 else
1975 imp->schedule_delayed_file(id, dat);
1976}
1977
1978void
1979database::put_file_version(file_id const & old_id,
1980 file_id const & new_id,
1981 file_delta const & del)
1982{
1983 I(!(old_id == new_id));
1984 file_data old_data, new_data;
1985 file_delta reverse_delta;
1986
1987 if (!file_version_exists(old_id))
1988 {
1989 W(F("file preimage '%s' missing in db") % old_id);
1990 W(F("dropping delta '%s' -> '%s'") % old_id % new_id);
1991 return;
1992 }
1993
1994 get_file_version(old_id, old_data);
1995 {
1996 data tmp;
1997 patch(old_data.inner(), del.inner(), tmp);
1998 new_data = file_data(tmp);
1999 }
2000
2001 {
2002 string tmp;
2003 invert_xdelta(old_data.inner()(), del.inner()(), tmp);
2004 reverse_delta = file_delta(tmp);
2005 data old_tmp;
2006 patch(new_data.inner(), reverse_delta.inner(), old_tmp);
2007 // We already have the real old data, so compare the
2008 // reconstruction to that directly instead of hashing
2009 // the reconstruction and comparing hashes.
2010 I(old_tmp == old_data.inner());
2011 }
2012
2013 transaction_guard guard(*this);
2014 if (file_or_manifest_base_exists(old_id, "files"))
2015 {
2016 // descendent of a head version replaces the head, therefore old head
2017 // must be disposed of
2018 imp->drop_or_cancel_file(old_id);
2019 }
2020 if (!file_or_manifest_base_exists(new_id, "files"))
2021 {
2022 imp->schedule_delayed_file(new_id, new_data);
2023 imp->drop(new_id.inner(), "file_deltas");
2024 }
2025
2026 if (!imp->delta_exists(old_id, new_id, "file_deltas"))
2027 {
2028 put_file_delta(old_id, new_id, reverse_delta);
2029 guard.commit();
2030 }
2031}
2032
2033void
2034database::get_arbitrary_file_delta(file_id const & src_id,
2035 file_id const & dst_id,
2036 file_delta & del)
2037{
2038 delta dtmp;
2039 // Deltas stored in the database go from base -> id.
2040 results res;
2041 query q1("SELECT delta FROM file_deltas "
2042 "WHERE base = ? AND id = ?");
2043 imp->fetch(res, one_col, any_rows,
2044 q1 % blob(src_id.inner()()) % blob(dst_id.inner()()));
2045
2046 if (!res.empty())
2047 {
2048 // Exact hit: a plain delta from src -> dst.
2049 gzip<delta> del_packed(res[0][0]);
2050 decode_gzip(del_packed, dtmp);
2051 del = file_delta(dtmp);
2052 return;
2053 }
2054
2055 query q2("SELECT delta FROM file_deltas "
2056 "WHERE base = ? AND id = ?");
2057 imp->fetch(res, one_col, any_rows,
2058 q2 % blob(dst_id.inner()()) % blob(src_id.inner()()));
2059
2060 if (!res.empty())
2061 {
2062 // We have a delta from dst -> src; we need to
2063 // invert this to a delta from src -> dst.
2064 gzip<delta> del_packed(res[0][0]);
2065 decode_gzip(del_packed, dtmp);
2066 string fwd_delta;
2067 file_data dst;
2068 get_file_version(dst_id, dst);
2069 invert_xdelta(dst.inner()(), dtmp(), fwd_delta);
2070 del = file_delta(fwd_delta);
2071 return;
2072 }
2073
2074 // No deltas of use; just load both versions and diff.
2075 file_data fd1, fd2;
2076 get_file_version(src_id, fd1);
2077 get_file_version(dst_id, fd2);
2078 diff(fd1.inner(), fd2.inner(), dtmp);
2079 del = file_delta(dtmp);
2080}
2081
2082
2083void
2084database::get_revision_ancestry(rev_ancestry_map & graph)
2085{
2086 // share some storage
2087 id::symtab id_syms;
2088
2089 results res;
2090 graph.clear();
2091 imp->fetch(res, 2, any_rows,
2092 query("SELECT parent,child FROM revision_ancestry"));
2093 for (size_t i = 0; i < res.size(); ++i)
2094 graph.insert(make_pair(revision_id(res[i][0]),
2095 revision_id(res[i][1])));
2096}
2097
2098void
2099database::get_revision_parents(revision_id const & id,
2100 set<revision_id> & parents)
2101{
2102 I(!null_id(id));
2103 parent_id_map::iterator i = imp->parent_cache.find(id);
2104 if (i == imp->parent_cache.end())
2105 {
2106 results res;
2107 parents.clear();
2108 imp->fetch(res, one_col, any_rows,
2109 query("SELECT parent FROM revision_ancestry WHERE child = ?")
2110 % blob(id.inner()()));
2111 for (size_t i = 0; i < res.size(); ++i)
2112 parents.insert(revision_id(res[i][0]));
2113
2114 imp->parent_cache.insert(make_pair(id, parents));
2115 }
2116 else
2117 {
2118 parents = i->second;
2119 }
2120}
2121
2122void
2123database::get_revision_children(revision_id const & id,
2124 set<revision_id> & children)
2125{
2126 results res;
2127 children.clear();
2128 imp->fetch(res, one_col, any_rows,
2129 query("SELECT child FROM revision_ancestry WHERE parent = ?")
2130 % blob(id.inner()()));
2131 for (size_t i = 0; i < res.size(); ++i)
2132 children.insert(revision_id(res[i][0]));
2133}
2134
2135void
2136database::get_leaves(set<revision_id> & leaves)
2137{
2138 results res;
2139 leaves.clear();
2140 imp->fetch(res, one_col, any_rows,
2141 query("SELECT revisions.id FROM revisions "
2142 "LEFT JOIN revision_ancestry "
2143 "ON revisions.id = revision_ancestry.parent "
2144 "WHERE revision_ancestry.child IS null"));
2145 for (size_t i = 0; i < res.size(); ++i)
2146 leaves.insert(revision_id(res[i][0]));
2147}
2148
2149
2150void
2151database::get_revision_manifest(revision_id const & rid,
2152 manifest_id & mid)
2153{
2154 revision_t rev;
2155 get_revision(rid, rev);
2156 mid = rev.new_manifest;
2157}
2158
2159void
2160database::get_common_ancestors(std::set<revision_id> const & revs,
2161 std::set<revision_id> & common_ancestors)
2162{
2163 set<revision_id> ancestors, all_common_ancestors;
2164 vector<revision_id> frontier;
2165 for (set<revision_id>::const_iterator i = revs.begin();
2166 i != revs.end(); ++i)
2167 {
2168 I(revision_exists(*i));
2169 ancestors.clear();
2170 ancestors.insert(*i);
2171 frontier.push_back(*i);
2172 while (!frontier.empty())
2173 {
2174 revision_id rid = frontier.back();
2175 frontier.pop_back();
2176 if(!null_id(rid))
2177 {
2178 set<revision_id> parents;
2179 get_revision_parents(rid, parents);
2180 for (set<revision_id>::const_iterator i = parents.begin();
2181 i != parents.end(); ++i)
2182 {
2183 if (ancestors.find(*i) == ancestors.end())
2184 {
2185 frontier.push_back(*i);
2186 ancestors.insert(*i);
2187 }
2188 }
2189 }
2190 }
2191 if (all_common_ancestors.empty())
2192 all_common_ancestors = ancestors;
2193 else
2194 {
2195 set<revision_id> common;
2196 set_intersection(ancestors.begin(), ancestors.end(),
2197 all_common_ancestors.begin(), all_common_ancestors.end(),
2198 inserter(common, common.begin()));
2199 all_common_ancestors = common;
2200 }
2201 }
2202
2203 for (set<revision_id>::const_iterator i = all_common_ancestors.begin();
2204 i != all_common_ancestors.end(); ++i)
2205 {
2206 // FIXME: where do these null'ed IDs come from?
2207 if (null_id(*i)) continue;
2208 common_ancestors.insert(*i);
2209 }
2210}
2211
2212void
2213database::get_revision(revision_id const & id,
2214 revision_t & rev)
2215{
2216 revision_data d;
2217 get_revision(id, d);
2218 read_revision(d, rev);
2219}
2220
2221void
2222database::get_revision(revision_id const & id,
2223 revision_data & dat)
2224{
2225 I(!null_id(id));
2226 results res;
2227 imp->fetch(res, one_col, one_row,
2228 query("SELECT data FROM revisions WHERE id = ?")
2229 % blob(id.inner()()));
2230
2231 gzip<data> gzdata(res[0][0]);
2232 data rdat;
2233 decode_gzip(gzdata,rdat);
2234
2235 // verify that we got a revision with the right id
2236 {
2237 revision_id tmp;
2238 calculate_ident(revision_data(rdat), tmp);
2239 I(id == tmp);
2240 }
2241
2242 dat = revision_data(rdat);
2243}
2244
2245void
2246database::get_rev_height(revision_id const & id,
2247 rev_height & height)
2248{
2249 if (null_id(id))
2250 {
2251 height = rev_height::root_height();
2252 return;
2253 }
2254
2255 height_map::const_iterator i = imp->height_cache.find(id);
2256 if (i == imp->height_cache.end())
2257 {
2258 results res;
2259 imp->fetch(res, one_col, one_row,
2260 query("SELECT height FROM heights WHERE revision = ?")
2261 % blob(id.inner()()));
2262
2263 I(res.size() == 1);
2264
2265 height = rev_height(res[0][0]);
2266 imp->height_cache.insert(make_pair(id, height));
2267 }
2268 else
2269 {
2270 height = i->second;
2271 }
2272
2273 I(height.valid());
2274}
2275
2276void
2277database::put_rev_height(revision_id const & id,
2278 rev_height const & height)
2279{
2280 I(!null_id(id));
2281 I(revision_exists(id));
2282 I(height.valid());
2283
2284 imp->height_cache.erase(id);
2285
2286 imp->execute(query("INSERT INTO heights VALUES(?, ?)")
2287 % blob(id.inner()())
2288 % blob(height()));
2289}
2290
2291bool
2292database::has_rev_height(rev_height const & height)
2293{
2294 results res;
2295 imp->fetch(res, one_col, any_rows,
2296 query("SELECT height FROM heights WHERE height = ?")
2297 % blob(height()));
2298 I((res.size() == 1) || (res.size() == 0));
2299 return res.size() == 1;
2300}
2301
2302void
2303database::deltify_revision(revision_id const & rid)
2304{
2305 transaction_guard guard(*this);
2306 revision_t rev;
2307 MM(rev);
2308 MM(rid);
2309 get_revision(rid, rev);
2310 // Make sure that all parent revs have their files replaced with deltas
2311 // from this rev's files.
2312 {
2313 for (edge_map::const_iterator i = rev.edges.begin();
2314 i != rev.edges.end(); ++i)
2315 {
2316 for (map<file_path, pair<file_id, file_id> >::const_iterator
2317 j = edge_changes(i).deltas_applied.begin();
2318 j != edge_changes(i).deltas_applied.end(); ++j)
2319 {
2320 if (file_or_manifest_base_exists(delta_entry_src(j), "files") &&
2321 file_version_exists(delta_entry_dst(j)))
2322 {
2323 file_data old_data;
2324 file_data new_data;
2325 get_file_version(delta_entry_src(j), old_data);
2326 get_file_version(delta_entry_dst(j), new_data);
2327 delta delt;
2328 diff(old_data.inner(), new_data.inner(), delt);
2329 file_delta del(delt);
2330 imp->drop_or_cancel_file(delta_entry_dst(j));
2331 imp->drop(delta_entry_dst(j).inner(), "file_deltas");
2332 put_file_version(delta_entry_src(j), delta_entry_dst(j), del);
2333 }
2334 }
2335 }
2336 }
2337 guard.commit();
2338}
2339
2340
2341bool
2342database::put_revision(revision_id const & new_id,
2343 revision_t const & rev)
2344{
2345 MM(new_id);
2346 MM(rev);
2347
2348 I(!null_id(new_id));
2349
2350 if (revision_exists(new_id))
2351 {
2352 if (global_sanity.debug_p())
2353 L(FL("revision '%s' already exists in db") % new_id);
2354 return false;
2355 }
2356
2357 I(rev.made_for == made_for_database);
2358 rev.check_sane();
2359
2360 // Phase 1: confirm the revision makes sense, and we the required files
2361 // actually exist
2362 for (edge_map::const_iterator i = rev.edges.begin();
2363 i != rev.edges.end(); ++i)
2364 {
2365 if (!edge_old_revision(i).inner()().empty()
2366 && !revision_exists(edge_old_revision(i)))
2367 {
2368 W(F("missing prerequisite revision '%s'")
2369 % edge_old_revision(i));
2370 W(F("dropping revision '%s'") % new_id);
2371 return false;
2372 }
2373
2374 for (map<file_path, file_id>::const_iterator a
2375 = edge_changes(i).files_added.begin();
2376 a != edge_changes(i).files_added.end(); ++a)
2377 {
2378 if (! file_version_exists(a->second))
2379 {
2380 W(F("missing prerequisite file '%s'") % a->second);
2381 W(F("dropping revision '%s'") % new_id);
2382 return false;
2383 }
2384 }
2385
2386 for (map<file_path, pair<file_id, file_id> >::const_iterator d
2387 = edge_changes(i).deltas_applied.begin();
2388 d != edge_changes(i).deltas_applied.end(); ++d)
2389 {
2390 I(!delta_entry_src(d).inner()().empty());
2391 I(!delta_entry_dst(d).inner()().empty());
2392
2393 if (! file_version_exists(delta_entry_src(d)))
2394 {
2395 W(F("missing prerequisite file pre-delta '%s'")
2396 % delta_entry_src(d));
2397 W(F("dropping revision '%s'") % new_id);
2398 return false;
2399 }
2400
2401 if (! file_version_exists(delta_entry_dst(d)))
2402 {
2403 W(F("missing prerequisite file post-delta '%s'")
2404 % delta_entry_dst(d));
2405 W(F("dropping revision '%s'") % new_id);
2406 return false;
2407 }
2408 }
2409 }
2410
2411 transaction_guard guard(*this);
2412
2413 // Phase 2: Write the revision data (inside a transaction)
2414
2415 revision_data d;
2416 write_revision(rev, d);
2417 gzip<data> d_packed;
2418 encode_gzip(d.inner(), d_packed);
2419 imp->execute(query("INSERT INTO revisions VALUES(?, ?)")
2420 % blob(new_id.inner()())
2421 % blob(d_packed()));
2422
2423 for (edge_map::const_iterator e = rev.edges.begin();
2424 e != rev.edges.end(); ++e)
2425 {
2426 imp->execute(query("INSERT INTO revision_ancestry VALUES(?, ?)")
2427 % blob(edge_old_revision(e).inner()())
2428 % blob(new_id.inner()()));
2429 }
2430 // We don't have to clear out the child's entry in the parent_cache,
2431 // because the child did not exist before this function was called, so
2432 // it can't be in the parent_cache already.
2433
2434 // Phase 3: Construct and write the roster (which also checks the manifest
2435 // id as it goes), but only if the roster does not already exist in the db
2436 // (i.e. because it was left over by a kill_rev_locally)
2437 // FIXME: there is no knowledge yet on speed implications for commands which
2438 // put a lot of revisions in a row (i.e. tailor or cvs_import)!
2439
2440 if (!roster_version_exists(new_id))
2441 {
2442 put_roster_for_revision(new_id, rev);
2443 }
2444 else
2445 {
2446 L(FL("roster for revision '%s' already exists in db") % new_id);
2447 }
2448
2449 // Phase 4: rewrite any files that need deltas added
2450
2451 deltify_revision(new_id);
2452
2453 // Phase 5: determine the revision height
2454
2455 put_height_for_revision(new_id, rev);
2456
2457 // Finally, commit.
2458
2459 guard.commit();
2460 return true;
2461}
2462
2463void
2464database::put_height_for_revision(revision_id const & new_id,
2465 revision_t const & rev)
2466{
2467 I(!null_id(new_id));
2468
2469 rev_height highest_parent;
2470 // we always branch off the highest parent ...
2471 for (edge_map::const_iterator e = rev.edges.begin();
2472 e != rev.edges.end(); ++e)
2473 {
2474 rev_height parent; MM(parent);
2475 get_rev_height(edge_old_revision(e), parent);
2476 if (parent > highest_parent)
2477 {
2478 highest_parent = parent;
2479 }
2480 }
2481
2482 // ... then find the first unused child
2483 u32 childnr(0);
2484 rev_height candidate; MM(candidate);
2485 while(true)
2486 {
2487 candidate = highest_parent.child_height(childnr);
2488 if (!has_rev_height(candidate))
2489 {
2490 break;
2491 }
2492 I(childnr < std::numeric_limits<u32>::max());
2493 ++childnr;
2494 }
2495 put_rev_height(new_id, candidate);
2496}
2497
2498void
2499database::put_roster_for_revision(revision_id const & new_id,
2500 revision_t const & rev)
2501{
2502 // Construct, the roster, sanity-check the manifest id, and then write it
2503 // to the db
2504 shared_ptr<roster_t> ros_writeable(new roster_t); MM(*ros_writeable);
2505 shared_ptr<marking_map> mm_writeable(new marking_map); MM(*mm_writeable);
2506 manifest_id roster_manifest_id;
2507 MM(roster_manifest_id);
2508 make_roster_for_revision(*this, rev, new_id, *ros_writeable, *mm_writeable);
2509 calculate_ident(*ros_writeable, roster_manifest_id);
2510 I(rev.new_manifest == roster_manifest_id);
2511 // const'ify the objects, suitable for caching etc.
2512 roster_t_cp ros = ros_writeable;
2513 marking_map_cp mm = mm_writeable;
2514 put_roster(new_id, ros, mm);
2515}
2516
2517bool
2518database::put_revision(revision_id const & new_id,
2519 revision_data const & dat)
2520{
2521 revision_t rev;
2522 read_revision(dat, rev);
2523 return put_revision(new_id, rev);
2524}
2525
2526
2527void
2528database::delete_existing_revs_and_certs()
2529{
2530 imp->execute(query("DELETE FROM revisions"));
2531 imp->execute(query("DELETE FROM revision_ancestry"));
2532 imp->execute(query("DELETE FROM revision_certs"));
2533}
2534
2535void
2536database::delete_existing_manifests()
2537{
2538 imp->execute(query("DELETE FROM manifests"));
2539 imp->execute(query("DELETE FROM manifest_deltas"));
2540}
2541
2542void
2543database::delete_existing_rosters()
2544{
2545 imp->execute(query("DELETE FROM rosters"));
2546 imp->execute(query("DELETE FROM roster_deltas"));
2547 imp->execute(query("DELETE FROM next_roster_node_number"));
2548}
2549
2550void
2551database::delete_existing_heights()
2552{
2553 imp->execute(query("DELETE FROM heights"));
2554}
2555
2556/// Deletes one revision from the local database.
2557/// @see kill_rev_locally
2558void
2559database::delete_existing_rev_and_certs(revision_id const & rid)
2560{
2561 transaction_guard guard (*this);
2562
2563 // Check that the revision exists and doesn't have any children.
2564 I(revision_exists(rid));
2565 set<revision_id> children;
2566 get_revision_children(rid, children);
2567 I(children.empty());
2568
2569
2570 L(FL("Killing revision %s locally") % rid);
2571
2572 // Kill the certs, ancestry, and revision.
2573 imp->execute(query("DELETE from revision_certs WHERE id = ?")
2574 % blob(rid.inner()()));
2575 imp->cert_stamper.note_change();
2576
2577 imp->execute(query("DELETE from revision_ancestry WHERE child = ?")
2578 % blob(rid.inner()()));
2579
2580 imp->execute(query("DELETE from heights WHERE revision = ?")
2581 % blob(rid.inner()()));
2582
2583 imp->execute(query("DELETE from revisions WHERE id = ?")
2584 % blob(rid.inner()()));
2585
2586 guard.commit();
2587}
2588
2589/// Deletes all certs referring to a particular branch.
2590void
2591database::delete_branch_named(cert_value const & branch)
2592{
2593 L(FL("Deleting all references to branch %s") % branch);
2594 imp->execute(query("DELETE FROM revision_certs WHERE name='branch' AND value =?")
2595 % blob(branch()));
2596 imp->cert_stamper.note_change();
2597 imp->execute(query("DELETE FROM branch_epochs WHERE branch=?")
2598 % blob(branch()));
2599}
2600
2601/// Deletes all certs referring to a particular tag.
2602void
2603database::delete_tag_named(cert_value const & tag)
2604{
2605 L(FL("Deleting all references to tag %s") % tag);
2606 imp->execute(query("DELETE FROM revision_certs WHERE name='tag' AND value =?")
2607 % blob(tag()));
2608 imp->cert_stamper.note_change();
2609}
2610
2611// crypto key management
2612
2613void
2614database::get_key_ids(vector<rsa_keypair_id> & pubkeys)
2615{
2616 pubkeys.clear();
2617 results res;
2618
2619 imp->fetch(res, one_col, any_rows, query("SELECT id FROM public_keys"));
2620
2621 for (size_t i = 0; i < res.size(); ++i)
2622 pubkeys.push_back(rsa_keypair_id(res[i][0]));
2623}
2624
2625void
2626database::get_key_ids(globish const & pattern,
2627 vector<rsa_keypair_id> & pubkeys)
2628{
2629 pubkeys.clear();
2630 results res;
2631
2632 imp->fetch(res, one_col, any_rows, query("SELECT id FROM public_keys"));
2633
2634 for (size_t i = 0; i < res.size(); ++i)
2635 if (pattern.matches(res[i][0]))
2636 pubkeys.push_back(rsa_keypair_id(res[i][0]));
2637}
2638
2639void
2640database_impl::get_keys(string const & table, vector<rsa_keypair_id> & keys)
2641{
2642 keys.clear();
2643 results res;
2644 fetch(res, one_col, any_rows, query("SELECT id FROM " + table));
2645 for (size_t i = 0; i < res.size(); ++i)
2646 keys.push_back(rsa_keypair_id(res[i][0]));
2647}
2648
2649void
2650database::get_public_keys(vector<rsa_keypair_id> & keys)
2651{
2652 imp->get_keys("public_keys", keys);
2653}
2654
2655bool
2656database::public_key_exists(id const & hash)
2657{
2658 results res;
2659 imp->fetch(res, one_col, any_rows,
2660 query("SELECT id FROM public_keys WHERE hash = ?")
2661 % blob(hash()));
2662 I((res.size() == 1) || (res.size() == 0));
2663 if (res.size() == 1)
2664 return true;
2665 return false;
2666}
2667
2668bool
2669database::public_key_exists(rsa_keypair_id const & id)
2670{
2671 results res;
2672 imp->fetch(res, one_col, any_rows,
2673 query("SELECT id FROM public_keys WHERE id = ?")
2674 % text(id()));
2675 I((res.size() == 1) || (res.size() == 0));
2676 if (res.size() == 1)
2677 return true;
2678 return false;
2679}
2680
2681void
2682database::get_pubkey(id const & hash,
2683 rsa_keypair_id & id,
2684 rsa_pub_key & pub)
2685{
2686 results res;
2687 imp->fetch(res, 2, one_row,
2688 query("SELECT id, keydata FROM public_keys WHERE hash = ?")
2689 % blob(hash()));
2690 id = rsa_keypair_id(res[0][0]);
2691 pub = rsa_pub_key(res[0][1]);
2692}
2693
2694void
2695database::get_key(rsa_keypair_id const & pub_id,
2696 rsa_pub_key & pub)
2697{
2698 results res;
2699 imp->fetch(res, one_col, one_row,
2700 query("SELECT keydata FROM public_keys WHERE id = ?")
2701 % text(pub_id()));
2702 pub = rsa_pub_key(res[0][0]);
2703}
2704
2705bool
2706database::put_key(rsa_keypair_id const & pub_id,
2707 rsa_pub_key const & pub)
2708{
2709 if (public_key_exists(pub_id))
2710 {
2711 rsa_pub_key tmp;
2712 get_key(pub_id, tmp);
2713 if (!keys_match(pub_id, tmp, pub_id, pub))
2714 W(F("key '%s' is not equal to key '%s' in database") % pub_id % pub_id);
2715 L(FL("skipping existing public key %s") % pub_id);
2716 return false;
2717 }
2718
2719 L(FL("putting public key %s") % pub_id);
2720
2721 id thash;
2722 key_hash_code(pub_id, pub, thash);
2723 I(!public_key_exists(thash));
2724
2725 imp->execute(query("INSERT INTO public_keys VALUES(?, ?, ?)")
2726 % blob(thash())
2727 % text(pub_id())
2728 % blob(pub()));
2729
2730 return true;
2731}
2732
2733void
2734database::delete_public_key(rsa_keypair_id const & pub_id)
2735{
2736 imp->execute(query("DELETE FROM public_keys WHERE id = ?")
2737 % text(pub_id()));
2738}
2739
2740void
2741database::encrypt_rsa(rsa_keypair_id const & pub_id,
2742 string const & plaintext,
2743 rsa_oaep_sha_data & ciphertext)
2744{
2745 rsa_pub_key pub;
2746 get_key(pub_id, pub);
2747
2748 SecureVector<Botan::byte> pub_block;
2749 pub_block.set(reinterpret_cast<Botan::byte const *>(pub().data()),
2750 pub().size());
2751
2752 shared_ptr<X509_PublicKey> x509_key(Botan::X509::load_key(pub_block));
2753 shared_ptr<RSA_PublicKey> pub_key
2754 = shared_dynamic_cast<RSA_PublicKey>(x509_key);
2755 if (!pub_key)
2756 throw informative_failure("Failed to get RSA encrypting key");
2757
2758 shared_ptr<PK_Encryptor>
2759 encryptor(get_pk_encryptor(*pub_key, "EME1(SHA-1)"));
2760
2761 SecureVector<Botan::byte> ct;
2762 ct = encryptor->encrypt(
2763 reinterpret_cast<Botan::byte const *>(plaintext.data()),
2764 plaintext.size());
2765 ciphertext = rsa_oaep_sha_data(string(reinterpret_cast<char const *>(ct.begin()),
2766 ct.size()));
2767}
2768
2769cert_status
2770database::check_signature(rsa_keypair_id const & id,
2771 string const & alleged_text,
2772 rsa_sha1_signature const & signature)
2773{
2774 shared_ptr<PK_Verifier> verifier;
2775
2776 verifier_cache::const_iterator i = imp->verifiers.find(id);
2777 if (i != imp->verifiers.end())
2778 verifier = i->second.first;
2779
2780 else
2781 {
2782 rsa_pub_key pub;
2783 SecureVector<Botan::byte> pub_block;
2784
2785 if (!public_key_exists(id))
2786 return cert_unknown;
2787
2788 get_key(id, pub);
2789 pub_block.set(reinterpret_cast<Botan::byte const *>(pub().data()),
2790 pub().size());
2791
2792 L(FL("building verifier for %d-byte pub key") % pub_block.size());
2793 shared_ptr<X509_PublicKey> x509_key(Botan::X509::load_key(pub_block));
2794 shared_ptr<RSA_PublicKey> pub_key
2795 = boost::shared_dynamic_cast<RSA_PublicKey>(x509_key);
2796
2797 E(pub_key,
2798 F("Failed to get RSA verifying key for %s") % id);
2799
2800 verifier.reset(get_pk_verifier(*pub_key, "EMSA3(SHA-1)"));
2801
2802 /* XXX This is ugly. We need to keep the key around
2803 * as long as the verifier is around, but the shared_ptr will go
2804 * away after we leave this scope. Hence we store a pair of
2805 * <verifier,key> so they both exist. */
2806 imp->verifiers.insert(make_pair(id, make_pair(verifier, pub_key)));
2807 }
2808
2809 // check the text+sig against the key
2810 L(FL("checking %d-byte signature") % signature().size());
2811
2812 if (verifier->verify_message(
2813 reinterpret_cast<Botan::byte const*>(alleged_text.data()),
2814 alleged_text.size(),
2815 reinterpret_cast<Botan::byte const*>(signature().data()),
2816 signature().size()))
2817 return cert_ok;
2818 else
2819 return cert_bad;
2820}
2821
2822// cert management
2823
2824bool
2825database_impl::cert_exists(cert const & t,
2826 string const & table)
2827{
2828 results res;
2829 query q = query("SELECT id FROM " + table + " WHERE id = ? "
2830 "AND name = ? "
2831 "AND value = ? "
2832 "AND keypair = ? "
2833 "AND signature = ?")
2834 % blob(t.ident.inner()())
2835 % text(t.name())
2836 % blob(t.value())
2837 % text(t.key())
2838 % blob(t.sig());
2839
2840 fetch(res, 1, any_rows, q);
2841
2842 I(res.size() == 0 || res.size() == 1);
2843 return res.size() == 1;
2844}
2845
2846void
2847database_impl::put_cert(cert const & t,
2848 string const & table)
2849{
2850 id thash;
2851 cert_hash_code(t, thash);
2852 rsa_sha1_signature sig;
2853
2854 string insert = "INSERT INTO " + table + " VALUES(?, ?, ?, ?, ?, ?)";
2855
2856 execute(query(insert)
2857 % blob(thash())
2858 % blob(t.ident.inner()())
2859 % text(t.name())
2860 % blob(t.value())
2861 % text(t.key())
2862 % blob(t.sig()));
2863}
2864
2865void
2866database_impl::results_to_certs(results const & res,
2867 vector<cert> & certs)
2868{
2869 certs.clear();
2870 for (size_t i = 0; i < res.size(); ++i)
2871 {
2872 cert t;
2873 t = cert(revision_id(res[i][0]),
2874 cert_name(res[i][1]),
2875 cert_value(res[i][2]),
2876 rsa_keypair_id(res[i][3]),
2877 rsa_sha1_signature(res[i][4]));
2878 certs.push_back(t);
2879 }
2880}
2881
2882void
2883database_impl::install_functions()
2884{
2885 // register any functions we're going to use
2886 I(sqlite3_create_function(sql(), "gunzip", -1,
2887 SQLITE_UTF8, NULL,
2888 &sqlite3_gunzip_fn,
2889 NULL, NULL) == 0);
2890}
2891
2892void
2893database_impl::get_certs(vector<cert> & certs,
2894 string const & table)
2895{
2896 results res;
2897 query q("SELECT id, name, value, keypair, signature FROM " + table);
2898 fetch(res, 5, any_rows, q);
2899 results_to_certs(res, certs);
2900}
2901
2902
2903void
2904database_impl::get_certs(id const & ident,
2905 vector<cert> & certs,
2906 string const & table)
2907{
2908 results res;
2909 query q("SELECT id, name, value, keypair, signature FROM " + table +
2910 " WHERE id = ?");
2911
2912 fetch(res, 5, any_rows, q % blob(ident()));
2913 results_to_certs(res, certs);
2914}
2915
2916
2917void
2918database_impl::get_certs(cert_name const & name,
2919 vector<cert> & certs,
2920 string const & table)
2921{
2922 results res;
2923 query q("SELECT id, name, value, keypair, signature FROM " + table +
2924 " WHERE name = ?");
2925 fetch(res, 5, any_rows, q % text(name()));
2926 results_to_certs(res, certs);
2927}
2928
2929
2930void
2931database_impl::get_certs(id const & ident,
2932 cert_name const & name,
2933 vector<cert> & certs,
2934 string const & table)
2935{
2936 results res;
2937 query q("SELECT id, name, value, keypair, signature FROM " + table +
2938 " WHERE id = ? AND name = ?");
2939
2940 fetch(res, 5, any_rows,
2941 q % blob(ident())
2942 % text(name()));
2943 results_to_certs(res, certs);
2944}
2945
2946void
2947database_impl::get_certs(cert_name const & name,
2948 cert_value const & val,
2949 vector<cert> & certs,
2950 string const & table)
2951{
2952 results res;
2953 query q("SELECT id, name, value, keypair, signature FROM " + table +
2954 " WHERE name = ? AND value = ?");
2955
2956 fetch(res, 5, any_rows,
2957 q % text(name())
2958 % blob(val()));
2959 results_to_certs(res, certs);
2960}
2961
2962
2963void
2964database_impl::get_certs(id const & ident,
2965 cert_name const & name,
2966 cert_value const & value,
2967 vector<cert> & certs,
2968 string const & table)
2969{
2970 results res;
2971 query q("SELECT id, name, value, keypair, signature FROM " + table +
2972 " WHERE id = ? AND name = ? AND value = ?");
2973
2974 fetch(res, 5, any_rows,
2975 q % blob(ident())
2976 % text(name())
2977 % blob(value()));
2978 results_to_certs(res, certs);
2979}
2980
2981
2982
2983bool
2984database::revision_cert_exists(revision<cert> const & cert)
2985{
2986 return imp->cert_exists(cert.inner(), "revision_certs");
2987}
2988
2989bool
2990database::put_revision_cert(revision<cert> const & cert)
2991{
2992 if (revision_cert_exists(cert))
2993 {
2994 L(FL("revision cert on '%s' already exists in db")
2995 % cert.inner().ident);
2996 return false;
2997 }
2998
2999 if (!revision_exists(revision_id(cert.inner().ident)))
3000 {
3001 W(F("cert revision '%s' does not exist in db")
3002 % cert.inner().ident);
3003 W(F("dropping cert"));
3004 return false;
3005 }
3006
3007 imp->put_cert(cert.inner(), "revision_certs");
3008 imp->cert_stamper.note_change();
3009 return true;
3010}
3011
3012outdated_indicator
3013database::get_revision_cert_nobranch_index(vector< pair<revision_id,
3014 pair<revision_id, rsa_keypair_id> > > & idx)
3015{
3016 // share some storage
3017 id::symtab id_syms;
3018
3019 results res;
3020 imp->fetch(res, 3, any_rows,
3021 query("SELECT hash, id, keypair "
3022 "FROM 'revision_certs' WHERE name != 'branch'"));
3023
3024 idx.clear();
3025 idx.reserve(res.size());
3026 for (results::const_iterator i = res.begin(); i != res.end(); ++i)
3027 {
3028 idx.push_back(make_pair(revision_id((*i)[0]),
3029 make_pair(revision_id((*i)[1]),
3030 rsa_keypair_id((*i)[2]))));
3031 }
3032 return imp->cert_stamper.get_indicator();
3033}
3034
3035outdated_indicator
3036database::get_revision_certs(vector< revision<cert> > & ts)
3037{
3038 vector<cert> certs;
3039 imp->get_certs(certs, "revision_certs");
3040 ts.clear();
3041 add_decoration_to_container(certs, ts);
3042 return imp->cert_stamper.get_indicator();
3043}
3044
3045outdated_indicator
3046database::get_revision_certs(cert_name const & name,
3047 vector< revision<cert> > & ts)
3048{
3049 vector<cert> certs;
3050 imp->get_certs(name, certs, "revision_certs");
3051 ts.clear();
3052 add_decoration_to_container(certs, ts);
3053 return imp->cert_stamper.get_indicator();
3054}
3055
3056outdated_indicator
3057database::get_revision_certs(revision_id const & id,
3058 cert_name const & name,
3059 vector< revision<cert> > & ts)
3060{
3061 vector<cert> certs;
3062 imp->get_certs(id.inner(), name, certs, "revision_certs");
3063 ts.clear();
3064 add_decoration_to_container(certs, ts);
3065 return imp->cert_stamper.get_indicator();
3066}
3067
3068outdated_indicator
3069database::get_revision_certs(revision_id const & id,
3070 cert_name const & name,
3071 cert_value const & val,
3072 vector< revision<cert> > & ts)
3073{
3074 vector<cert> certs;
3075 imp->get_certs(id.inner(), name, val, certs, "revision_certs");
3076 ts.clear();
3077 add_decoration_to_container(certs, ts);
3078 return imp->cert_stamper.get_indicator();
3079}
3080
3081outdated_indicator
3082database::get_revisions_with_cert(cert_name const & name,
3083 cert_value const & val,
3084 set<revision_id> & revisions)
3085{
3086 revisions.clear();
3087 results res;
3088 query q("SELECT id FROM revision_certs WHERE name = ? AND value = ?");
3089 imp->fetch(res, one_col, any_rows, q % text(name()) % blob(val()));
3090 for (results::const_iterator i = res.begin(); i != res.end(); ++i)
3091 revisions.insert(revision_id((*i)[0]));
3092 return imp->cert_stamper.get_indicator();
3093}
3094
3095outdated_indicator
3096database::get_revision_certs(cert_name const & name,
3097 cert_value const & val,
3098 vector< revision<cert> > & ts)
3099{
3100 vector<cert> certs;
3101 imp->get_certs(name, val, certs, "revision_certs");
3102 ts.clear();
3103 add_decoration_to_container(certs, ts);
3104 return imp->cert_stamper.get_indicator();
3105}
3106
3107outdated_indicator
3108database::get_revision_certs(revision_id const & id,
3109 vector< revision<cert> > & ts)
3110{
3111 vector<cert> certs;
3112 imp->get_certs(id.inner(), certs, "revision_certs");
3113 ts.clear();
3114 add_decoration_to_container(certs, ts);
3115 return imp->cert_stamper.get_indicator();
3116}
3117
3118outdated_indicator
3119database::get_revision_certs(revision_id const & ident,
3120 vector<id> & ts)
3121{
3122 results res;
3123 vector<cert> certs;
3124 imp->fetch(res, one_col, any_rows,
3125 query("SELECT hash "
3126 "FROM revision_certs "
3127 "WHERE id = ?")
3128 % blob(ident.inner()()));
3129 ts.clear();
3130 for (size_t i = 0; i < res.size(); ++i)
3131 ts.push_back(id(res[i][0]));
3132 return imp->cert_stamper.get_indicator();
3133}
3134
3135void
3136database::get_revision_cert(id const & hash,
3137 revision<cert> & c)
3138{
3139 results res;
3140 vector<cert> certs;
3141 imp->fetch(res, 5, one_row,
3142 query("SELECT id, name, value, keypair, signature "
3143 "FROM revision_certs "
3144 "WHERE hash = ?")
3145 % blob(hash()));
3146 imp->results_to_certs(res, certs);
3147 I(certs.size() == 1);
3148 c = revision<cert>(certs[0]);
3149}
3150
3151bool
3152database::revision_cert_exists(revision_id const & hash)
3153{
3154 results res;
3155 vector<cert> certs;
3156 imp->fetch(res, one_col, any_rows,
3157 query("SELECT id "
3158 "FROM revision_certs "
3159 "WHERE hash = ?")
3160 % blob(hash.inner()()));
3161 I(res.size() == 0 || res.size() == 1);
3162 return (res.size() == 1);
3163}
3164
3165void
3166database::get_manifest_certs(manifest_id const & id,
3167 vector< manifest<cert> > & ts)
3168{
3169 vector<cert> certs;
3170 imp->get_certs(id.inner(), certs, "manifest_certs");
3171 ts.clear();
3172 add_decoration_to_container(certs, ts);
3173}
3174
3175
3176void
3177database::get_manifest_certs(cert_name const & name,
3178 vector< manifest<cert> > & ts)
3179{
3180 vector<cert> certs;
3181 imp->get_certs(name, certs, "manifest_certs");
3182 ts.clear();
3183 add_decoration_to_container(certs, ts);
3184}
3185
3186
3187// completions
3188void
3189database_impl::add_prefix_matching_constraint(string const & colname,
3190 string const & prefix,
3191 query & q)
3192{
3193 L(FL("add_prefix_matching_constraint for '%s'") % prefix);
3194
3195 if (prefix.empty())
3196 q.sql_cmd += "1"; // always true
3197 else
3198 {
3199 string binary_prefix = decode_hexenc(prefix);
3200 string lower_bound(binary_prefix);
3201 string upper_bound(binary_prefix);
3202
3203 string::reverse_iterator ity(upper_bound.rbegin());
3204 ++(*ity);
3205 while ((*ity == 0) && ity != upper_bound.rend())
3206 {
3207 ++ity;
3208 ++(*ity);
3209 }
3210
3211 if (ity == upper_bound.rend())
3212 {
3213 // no upper bound needed, as the lower bound is
3214 // 0xffffff...
3215 if (global_sanity.debug_p())
3216 L(FL("prefix_matcher: only lower bound ('%s')")
3217 % encode_hexenc(lower_bound));
3218
3219 q.sql_cmd += colname + " > ?";
3220 q.args.push_back(blob(lower_bound));
3221 }
3222 else
3223 {
3224 if (global_sanity.debug_p())
3225 L(FL("prefix_matcher: lower bound ('%s') and upper bound ('%s')")
3226 % encode_hexenc(lower_bound)
3227 % encode_hexenc(upper_bound));
3228
3229 q.sql_cmd += colname + " BETWEEN ? AND ?";
3230 q.args.push_back(blob(lower_bound));
3231 q.args.push_back(blob(upper_bound));
3232 }
3233
3234 // encode_hexenc might have lost a nibble a the end, thus we possibly
3235 // need to add a second check, with a LIKE operator on the hex
3236 // encoded string.
3237
3238 if (prefix.size() % 2 == 1)
3239 {
3240 string pattern = prefix + '%';
3241 q.sql_cmd += " AND (hex(" + colname + ") LIKE ?)";
3242 q.args.push_back(text(pattern));
3243 }
3244 }
3245}
3246
3247void
3248database::complete(string const & partial,
3249 set<revision_id> & completions)
3250{
3251 results res;
3252 completions.clear();
3253 query q("SELECT id FROM revisions WHERE ");
3254
3255 imp->add_prefix_matching_constraint("id", partial, q);
3256 imp->fetch(res, 1, any_rows, q);
3257
3258 for (size_t i = 0; i < res.size(); ++i)
3259 completions.insert(revision_id(res[i][0]));
3260}
3261
3262
3263void
3264database::complete(string const & partial,
3265 set<file_id> & completions)
3266{
3267 results res;
3268 completions.clear();
3269
3270 query q("SELECT id FROM files WHERE ");
3271 imp->add_prefix_matching_constraint("id", partial, q);
3272 imp->fetch(res, 1, any_rows, q);
3273
3274 for (size_t i = 0; i < res.size(); ++i)
3275 completions.insert(file_id(res[i][0]));
3276
3277 res.clear();
3278
3279 q = query("SELECT id FROM file_deltas WHERE ");
3280 imp->add_prefix_matching_constraint("id", partial, q);
3281 imp->fetch(res, 1, any_rows, q);
3282
3283 for (size_t i = 0; i < res.size(); ++i)
3284 completions.insert(file_id(res[i][0]));
3285}
3286
3287void
3288database::complete(string const & partial,
3289 set< pair<key_id, utf8 > > & completions)
3290{
3291 results res;
3292 completions.clear();
3293 query q("SELECT hash, id FROM public_keys WHERE ");
3294
3295 imp->add_prefix_matching_constraint("hash", partial, q);
3296 imp->fetch(res, 2, any_rows, q);
3297
3298 for (size_t i = 0; i < res.size(); ++i)
3299 completions.insert(make_pair(key_id(res[i][0]),
3300 utf8(res[i][1])));
3301}
3302
3303// revision selectors
3304
3305void
3306database::select_parent(string const & partial,
3307 set<revision_id> & completions)
3308{
3309 results res;
3310 completions.clear();
3311
3312 query q("SELECT DISTINCT parent FROM revision_ancestry WHERE ");
3313 imp->add_prefix_matching_constraint("child", encode_hexenc(partial), q);
3314 imp->fetch(res, 1, any_rows, q);
3315
3316 for (size_t i = 0; i < res.size(); ++i)
3317 completions.insert(revision_id(res[i][0]));
3318}
3319
3320void
3321database::select_cert(string const & certname,
3322 set<revision_id> & completions)
3323{
3324 results res;
3325 completions.clear();
3326
3327 imp->fetch(res, 1, any_rows,
3328 query("SELECT DISTINCT id FROM revision_certs WHERE name = ?")
3329 % text(certname));
3330
3331 for (size_t i = 0; i < res.size(); ++i)
3332 completions.insert(revision_id(res[i][0]));
3333}
3334
3335void
3336database::select_cert(string const & certname, string const & certvalue,
3337 set<revision_id> & completions)
3338{
3339 results res;
3340 completions.clear();
3341
3342 imp->fetch(res, 1, any_rows,
3343 query("SELECT DISTINCT id FROM revision_certs"
3344 " WHERE name = ? AND CAST(value AS TEXT) GLOB ?")
3345 % text(certname) % text(certvalue));
3346
3347 for (size_t i = 0; i < res.size(); ++i)
3348 completions.insert(revision_id(res[i][0]));
3349}
3350
3351void
3352database::select_author_tag_or_branch(string const & partial,
3353 set<revision_id> & completions)
3354{
3355 results res;
3356 completions.clear();
3357
3358 string pattern = partial + "*";
3359
3360 imp->fetch(res, 1, any_rows,
3361 query("SELECT DISTINCT id FROM revision_certs"
3362 " WHERE (name=? OR name=? OR name=?)"
3363 " AND CAST(value AS TEXT) GLOB ?")
3364 % text(author_cert_name()) % text(tag_cert_name())
3365 % text(branch_cert_name()) % text(pattern));
3366
3367 for (size_t i = 0; i < res.size(); ++i)
3368 completions.insert(revision_id(res[i][0]));
3369}
3370
3371void
3372database::select_date(string const & date, string const & comparison,
3373 set<revision_id> & completions)
3374{
3375 results res;
3376 completions.clear();
3377
3378 query q;
3379 q.sql_cmd = ("SELECT DISTINCT id FROM revision_certs "
3380 "WHERE name = ? AND CAST(value AS TEXT) ");
3381 q.sql_cmd += comparison;
3382 q.sql_cmd += " ?";
3383
3384 imp->fetch(res, 1, any_rows,
3385 q % text(date_cert_name()) % text(date));
3386 for (size_t i = 0; i < res.size(); ++i)
3387 completions.insert(revision_id(res[i][0]));
3388}
3389
3390// epochs
3391
3392void
3393database::get_epochs(map<branch_name, epoch_data> & epochs)
3394{
3395 epochs.clear();
3396 results res;
3397 imp->fetch(res, 2, any_rows, query("SELECT branch, epoch FROM branch_epochs"));
3398 for (results::const_iterator i = res.begin(); i != res.end(); ++i)
3399 {
3400 branch_name decoded(idx(*i, 0));
3401 I(epochs.find(decoded) == epochs.end());
3402 epochs.insert(make_pair(decoded,
3403 epoch_data(idx(*i, 1))));
3404 }
3405}
3406
3407void
3408database::get_epoch(epoch_id const & eid,
3409 branch_name & branch, epoch_data & epo)
3410{
3411 I(epoch_exists(eid));
3412 results res;
3413 imp->fetch(res, 2, any_rows,
3414 query("SELECT branch, epoch FROM branch_epochs"
3415 " WHERE hash = ?")
3416 % blob(eid.inner()()));
3417 I(res.size() == 1);
3418 branch = branch_name(idx(idx(res, 0), 0));
3419 epo = epoch_data(idx(idx(res, 0), 1));
3420}
3421
3422bool
3423database::epoch_exists(epoch_id const & eid)
3424{
3425 results res;
3426 imp->fetch(res, one_col, any_rows,
3427 query("SELECT hash FROM branch_epochs WHERE hash = ?")
3428 % blob(eid.inner()()));
3429 I(res.size() == 1 || res.size() == 0);
3430 return res.size() == 1;
3431}
3432
3433void
3434database::set_epoch(branch_name const & branch, epoch_data const & epo)
3435{
3436 epoch_id eid;
3437 epoch_hash_code(branch, epo, eid);
3438 I(epo.inner()().size() == constants::epochlen_bytes);
3439 imp->execute(query("INSERT OR REPLACE INTO branch_epochs VALUES(?, ?, ?)")
3440 % blob(eid.inner()())
3441 % blob(branch())
3442 % blob(epo.inner()()));
3443}
3444
3445void
3446database::clear_epoch(branch_name const & branch)
3447{
3448 imp->execute(query("DELETE FROM branch_epochs WHERE branch = ?")
3449 % blob(branch()));
3450}
3451
3452bool
3453database::check_integrity()
3454{
3455 results res;
3456 imp->fetch(res, one_col, any_rows, query("PRAGMA integrity_check"));
3457 I(res.size() == 1);
3458 I(res[0].size() == 1);
3459
3460 return res[0][0] == "ok";
3461}
3462
3463// vars
3464
3465void
3466database::get_vars(map<var_key, var_value> & vars)
3467{
3468 vars.clear();
3469 results res;
3470 imp->fetch(res, 3, any_rows, query("SELECT domain, name, value FROM db_vars"));
3471 for (results::const_iterator i = res.begin(); i != res.end(); ++i)
3472 {
3473 var_domain domain(idx(*i, 0));
3474 var_name name(idx(*i, 1));
3475 var_value value(idx(*i, 2));
3476 I(vars.find(make_pair(domain, name)) == vars.end());
3477 vars.insert(make_pair(make_pair(domain, name), value));
3478 }
3479}
3480
3481void
3482database::get_var(var_key const & key, var_value & value)
3483{
3484 // FIXME: sillyly inefficient. Doesn't really matter, though.
3485 map<var_key, var_value> vars;
3486 get_vars(vars);
3487 map<var_key, var_value>::const_iterator i = vars.find(key);
3488 I(i != vars.end());
3489 value = i->second;
3490}
3491
3492bool
3493database::var_exists(var_key const & key)
3494{
3495 // FIXME: sillyly inefficient. Doesn't really matter, though.
3496 map<var_key, var_value> vars;
3497 get_vars(vars);
3498 map<var_key, var_value>::const_iterator i = vars.find(key);
3499 return i != vars.end();
3500}
3501
3502void
3503database::set_var(var_key const & key, var_value const & value)
3504{
3505 imp->execute(query("INSERT OR REPLACE INTO db_vars VALUES(?, ?, ?)")
3506 % text(key.first())
3507 % blob(key.second())
3508 % blob(value()));
3509}
3510
3511void
3512database::clear_var(var_key const & key)
3513{
3514 imp->execute(query("DELETE FROM db_vars WHERE domain = ? AND name = ?")
3515 % text(key.first())
3516 % blob(key.second()));
3517}
3518
3519// branches
3520
3521outdated_indicator
3522database::get_branches(vector<string> & names)
3523{
3524 results res;
3525 query q("SELECT DISTINCT value FROM revision_certs WHERE name = ?");
3526 string cert_name = "branch";
3527 imp->fetch(res, one_col, any_rows, q % text(cert_name));
3528 for (size_t i = 0; i < res.size(); ++i)
3529 {
3530 names.push_back(res[i][0]);
3531 }
3532 return imp->cert_stamper.get_indicator();
3533}
3534
3535outdated_indicator
3536database::get_branches(globish const & glob,
3537 vector<string> & names)
3538{
3539 results res;
3540 query q("SELECT DISTINCT value FROM revision_certs WHERE name = ?");
3541 string cert_name = "branch";
3542 imp->fetch(res, one_col, any_rows, q % text(cert_name));
3543 for (size_t i = 0; i < res.size(); ++i)
3544 {
3545 if (glob.matches(res[i][0]))
3546 names.push_back(res[i][0]);
3547 }
3548 return imp->cert_stamper.get_indicator();
3549}
3550
3551void
3552database::get_roster(revision_id const & rev_id,
3553 roster_t & roster)
3554{
3555 marking_map mm;
3556 get_roster(rev_id, roster, mm);
3557}
3558
3559void
3560database::get_roster(revision_id const & rev_id,
3561 roster_t & roster,
3562 marking_map & marking)
3563{
3564 if (rev_id.inner()().empty())
3565 {
3566 roster = roster_t();
3567 marking = marking_map();
3568 return;
3569 }
3570
3571 cached_roster cr;
3572 get_roster(rev_id, cr);
3573 roster = *cr.first;
3574 marking = *cr.second;
3575}
3576
3577void
3578database::get_roster(revision_id const & rev_id, cached_roster & cr)
3579{
3580 get_roster_version(rev_id, cr);
3581 I(cr.first);
3582 I(cr.second);
3583}
3584
3585void
3586database::put_roster(revision_id const & rev_id,
3587 roster_t_cp const & roster,
3588 marking_map_cp const & marking)
3589{
3590 I(roster);
3591 I(marking);
3592 MM(rev_id);
3593
3594 transaction_guard guard(*this);
3595
3596 // Our task is to add this roster, and deltify all the incoming edges (if
3597 // they aren't already).
3598
3599 imp->roster_cache.insert_dirty(rev_id, make_pair(roster, marking));
3600
3601 set<revision_id> parents;
3602 get_revision_parents(rev_id, parents);
3603
3604 // Now do what deltify would do if we bothered
3605 for (set<revision_id>::const_iterator i = parents.begin();
3606 i != parents.end(); ++i)
3607 {
3608 if (null_id(*i))
3609 continue;
3610 revision_id old_rev = *i;
3611 if (imp->roster_base_stored(old_rev))
3612 {
3613 cached_roster cr;
3614 get_roster_version(old_rev, cr);
3615 roster_delta reverse_delta;
3616 delta_rosters(*roster, *marking, *(cr.first), *(cr.second), reverse_delta);
3617 if (imp->roster_cache.exists(old_rev))
3618 imp->roster_cache.mark_clean(old_rev);
3619 imp->drop(old_rev.inner(), "rosters");
3620 imp->put_roster_delta(old_rev, rev_id, reverse_delta);
3621 }
3622 }
3623 guard.commit();
3624}
3625
3626// for get_uncommon_ancestors
3627struct rev_height_graph : rev_graph
3628{
3629 rev_height_graph(database & db) : db(db) {}
3630 virtual void get_parents(revision_id const & rev, set<revision_id> & parents) const
3631 {
3632 db.get_revision_parents(rev, parents);
3633 }
3634 virtual void get_children(revision_id const & rev, set<revision_id> & parents) const
3635 {
3636 // not required
3637 I(false);
3638 }
3639 virtual void get_height(revision_id const & rev, rev_height & h) const
3640 {
3641 db.get_rev_height(rev, h);
3642 }
3643
3644 database & db;
3645};
3646
3647void
3648database::get_uncommon_ancestors(revision_id const & a,
3649 revision_id const & b,
3650 set<revision_id> & a_uncommon_ancs,
3651 set<revision_id> & b_uncommon_ancs)
3652{
3653
3654 rev_height_graph graph(*this);
3655 ::get_uncommon_ancestors(a, b, graph, a_uncommon_ancs, b_uncommon_ancs);
3656}
3657
3658node_id
3659database::next_node_id()
3660{
3661 transaction_guard guard(*this);
3662 results res;
3663
3664 // We implement this as a fixed db var.
3665 imp->fetch(res, one_col, any_rows,
3666 query("SELECT node FROM next_roster_node_number"));
3667
3668 u64 n = 1;
3669 if (res.empty())
3670 {
3671 imp->execute(query("INSERT INTO next_roster_node_number VALUES(1)"));
3672 }
3673 else
3674 {
3675 I(res.size() == 1);
3676 n = lexical_cast<u64>(res[0][0]);
3677 ++n;
3678 imp->execute(query("UPDATE next_roster_node_number SET node = ?")
3679 % text(lexical_cast<string>(n)));
3680 }
3681 guard.commit();
3682 return static_cast<node_id>(n);
3683}
3684
3685void
3686database_impl::check_filename()
3687{
3688 N(!filename.empty(), F("no database specified"));
3689}
3690
3691
3692void
3693database_impl::check_db_exists()
3694{
3695 switch (get_path_status(filename))
3696 {
3697 case path::file:
3698 return;
3699
3700 case path::nonexistent:
3701 N(false, F("database %s does not exist") % filename);
3702
3703 case path::directory:
3704 if (directory_is_workspace(filename))
3705 {
3706 system_path workspace_database;
3707 workspace::get_database_option(filename, workspace_database);
3708 N(workspace_database.empty(),
3709 F("%s is a workspace, not a database\n"
3710 "(did you mean %s?)") % filename % workspace_database);
3711 }
3712 N(false, F("%s is a directory, not a database") % filename);
3713 }
3714}
3715
3716void
3717database_impl::check_db_nonexistent()
3718{
3719 require_path_is_nonexistent(filename,
3720 F("database %s already exists")
3721 % filename);
3722
3723 system_path journal(filename.as_internal() + "-journal");
3724 require_path_is_nonexistent(journal,
3725 F("existing (possibly stale) journal file '%s' "
3726 "has same stem as new database '%s'\n"
3727 "cancelling database creation")
3728 % journal % filename);
3729
3730}
3731
3732void
3733database_impl::open()
3734{
3735 I(!__sql);
3736
3737 if (sqlite3_open(filename.as_external().c_str(), &__sql) == SQLITE_NOMEM)
3738 throw std::bad_alloc();
3739
3740 I(__sql);
3741 assert_sqlite3_ok(__sql);
3742}
3743
3744void
3745database_impl::close()
3746{
3747 I(__sql);
3748
3749 sqlite3_close(__sql);
3750 __sql = 0;
3751
3752 I(!__sql);
3753}
3754
3755// the database holds onto the lua_hooks object and uses it to re-expose
3756// these two hooks. it is impractical to pass the lua_hooks object down to
3757// all the places where this is used, and they're all going to get
3758// reexamined when we do policy branches anyway. also, arguably this is
3759// cleaner, because those places don't have access to *all* the lua hooks,
3760// just these two. (manifest cert trust is only relevant to pre-roster
3761// migration, but revision cert trust comes up everywhere erase_bogus_certs
3762// is called.) (A near-term refactor that might make sense: make
3763// erase_bogus_certs a project_t member and have the project_t hold the
3764// lua_hooks reference.)
3765
3766bool
3767database::hook_get_manifest_cert_trust(set<rsa_keypair_id> const & signers,
3768 manifest_id const & id, cert_name const & name, cert_value const & val)
3769{
3770 return lua.hook_get_manifest_cert_trust(signers, id, name, val);
3771};
3772
3773bool
3774database::hook_get_revision_cert_trust(set<rsa_keypair_id> const & signers,
3775 revision_id const & id, cert_name const & name, cert_value const & val)
3776{
3777 return lua.hook_get_revision_cert_trust(signers, id, name, val);
3778};
3779
3780// transaction guards
3781
3782conditional_transaction_guard::~conditional_transaction_guard()
3783{
3784 if (!acquired)
3785 return;
3786 if (committed)
3787 db.imp->commit_transaction();
3788 else
3789 db.imp->rollback_transaction();
3790}
3791
3792void
3793conditional_transaction_guard::acquire()
3794{
3795 I(!acquired);
3796 acquired = true;
3797 db.imp->begin_transaction(exclusive);
3798}
3799
3800void
3801conditional_transaction_guard::do_checkpoint()
3802{
3803 I(acquired);
3804 db.imp->commit_transaction();
3805 db.imp->begin_transaction(exclusive);
3806 checkpointed_calls = 0;
3807 checkpointed_bytes = 0;
3808}
3809
3810void
3811conditional_transaction_guard::maybe_checkpoint(size_t nbytes)
3812{
3813 I(acquired);
3814 checkpointed_calls += 1;
3815 checkpointed_bytes += nbytes;
3816 if (checkpointed_calls >= checkpoint_batch_size
3817 || checkpointed_bytes >= checkpoint_batch_bytes)
3818 do_checkpoint();
3819}
3820
3821void
3822conditional_transaction_guard::commit()
3823{
3824 I(acquired);
3825 committed = true;
3826}
3827
3828
3829
3830// Local Variables:
3831// mode: C++
3832// fill-column: 76
3833// c-file-style: "gnu"
3834// indent-tabs-mode: nil
3835// End:
3836// vim: et:sw=2:sts=2:ts=2:cino=>2s,{s,\:s,+s,t0,g0,^-2,e-2,n-2,p2s,(0,=s:

Archive Download this file

Branches

Tags

Quick Links:     www.monotone.ca    -     Downloads    -     Documentation    -     Wiki    -     Code Forge    -     Build Status