env_chromium.h 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347
  1. // Copyright (c) 2013 The LevelDB Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file. See the AUTHORS file for names of contributors.
  4. #ifndef THIRD_PARTY_LEVELDATABASE_ENV_CHROMIUM_H_
  5. #define THIRD_PARTY_LEVELDATABASE_ENV_CHROMIUM_H_
  6. #include <memory>
  7. #include <set>
  8. #include <string>
  9. #include <vector>
  10. #include "base/callback.h"
  11. #include "base/containers/circular_deque.h"
  12. #include "base/containers/flat_map.h"
  13. #include "base/containers/linked_list.h"
  14. #include "base/files/file.h"
  15. #include "base/files/file_path.h"
  16. #include "base/gtest_prod_util.h"
  17. #include "base/macros.h"
  18. #include "base/synchronization/condition_variable.h"
  19. #include "build/build_config.h"
  20. #include "leveldb/cache.h"
  21. #include "leveldb/db.h"
  22. #include "leveldb/env.h"
  23. #include "leveldb/export.h"
  24. #include "port/port_chromium.h"
  25. #include "util/mutexlock.h"
  26. namespace base {
  27. namespace trace_event {
  28. class MemoryAllocatorDump;
  29. class ProcessMemoryDump;
  30. } // namespace trace_event
  31. } // namespace base
  32. namespace storage {
  33. class FilesystemProxy;
  34. }
  35. namespace leveldb_env {
  36. // These entries map to values in tools/metrics/histograms/histograms.xml. New
  37. // values should be appended at the end.
  38. enum MethodID {
  39. kSequentialFileRead,
  40. kSequentialFileSkip,
  41. kRandomAccessFileRead,
  42. kWritableFileAppend,
  43. kWritableFileClose,
  44. kWritableFileFlush,
  45. kWritableFileSync,
  46. kNewSequentialFile,
  47. kNewRandomAccessFile,
  48. kNewWritableFile,
  49. kObsoleteDeleteFile,
  50. kCreateDir,
  51. kObsoleteDeleteDir,
  52. kGetFileSize,
  53. kRenameFile,
  54. kLockFile,
  55. kUnlockFile,
  56. kGetTestDirectory,
  57. kNewLogger,
  58. kSyncParent,
  59. kGetChildren,
  60. kNewAppendableFile,
  61. kRemoveFile,
  62. kRemoveDir,
  63. kNumEntries
  64. };
  65. // leveldb::Status::Code values are mapped to these values for UMA logging.
  66. // Do not change/delete these values as you will break reporting for older
  67. // copies of Chrome. Only add new values to the end.
  68. enum LevelDBStatusValue {
  69. LEVELDB_STATUS_OK = 0,
  70. LEVELDB_STATUS_NOT_FOUND,
  71. LEVELDB_STATUS_CORRUPTION,
  72. LEVELDB_STATUS_NOT_SUPPORTED,
  73. LEVELDB_STATUS_INVALID_ARGUMENT,
  74. LEVELDB_STATUS_IO_ERROR,
  75. LEVELDB_STATUS_MAX
  76. };
  77. LEVELDB_EXPORT LevelDBStatusValue
  78. GetLevelDBStatusUMAValue(const leveldb::Status& s);
  79. using DatabaseErrorReportingCallback =
  80. base::RepeatingCallback<void(const leveldb::Status&)>;
  81. // Create the default leveldb options object suitable for leveldb operations.
  82. struct LEVELDB_EXPORT Options : public leveldb::Options {
  83. Options();
  84. // Called when there is a error during the Get() call. Intended for metrics
  85. // reporting.
  86. DatabaseErrorReportingCallback on_get_error;
  87. // Called when there is a error during the Write() call, which is called for
  88. // Write(), Put() and Delete(). Intended for metrics reporting.
  89. DatabaseErrorReportingCallback on_write_error;
  90. };
  91. LEVELDB_EXPORT const char* MethodIDToString(MethodID method);
  92. leveldb::Status LEVELDB_EXPORT MakeIOError(leveldb::Slice filename,
  93. const std::string& message,
  94. MethodID method,
  95. base::File::Error error);
  96. leveldb::Status LEVELDB_EXPORT MakeIOError(leveldb::Slice filename,
  97. const std::string& message,
  98. MethodID method);
  99. enum ErrorParsingResult {
  100. METHOD_ONLY,
  101. METHOD_AND_BFE,
  102. NONE,
  103. };
  104. ErrorParsingResult LEVELDB_EXPORT
  105. ParseMethodAndError(const leveldb::Status& status,
  106. MethodID* method,
  107. base::File::Error* error);
  108. LEVELDB_EXPORT int GetCorruptionCode(const leveldb::Status& status);
  109. LEVELDB_EXPORT int GetNumCorruptionCodes();
  110. LEVELDB_EXPORT std::string GetCorruptionMessage(const leveldb::Status& status);
  111. LEVELDB_EXPORT bool IndicatesDiskFull(const leveldb::Status& status);
  112. // Returns the name for a temporary database copy during RewriteDB().
  113. LEVELDB_EXPORT std::string DatabaseNameForRewriteDB(
  114. const std::string& original_name);
  115. // Determine the appropriate leveldb write buffer size to use. The default size
  116. // (4MB) may result in a log file too large to be compacted given the available
  117. // storage space. This function will return smaller values for smaller disks,
  118. // and the default leveldb value for larger disks.
  119. //
  120. // |disk_space| is the logical partition size (in bytes), and *not* available
  121. // space. A value of -1 will return leveldb's default write buffer size.
  122. LEVELDB_EXPORT extern size_t WriteBufferSize(int64_t disk_space);
  123. class LEVELDB_EXPORT ChromiumEnv : public leveldb::Env {
  124. public:
  125. using ScheduleFunc = void(void*);
  126. // Constructs a ChromiumEnv instance with an unrestricted FilesystemProxy
  127. // instance that performs direct filesystem access.
  128. ChromiumEnv();
  129. // Constructs a ChromiumEnv instance with a custom FilesystemProxy instance.
  130. explicit ChromiumEnv(std::unique_ptr<storage::FilesystemProxy> filesystem);
  131. ~ChromiumEnv() override;
  132. bool FileExists(const std::string& fname) override;
  133. leveldb::Status GetChildren(const std::string& dir,
  134. std::vector<std::string>* result) override;
  135. leveldb::Status RemoveFile(const std::string& fname) override;
  136. leveldb::Status CreateDir(const std::string& name) override;
  137. leveldb::Status RemoveDir(const std::string& name) override;
  138. leveldb::Status GetFileSize(const std::string& fname,
  139. uint64_t* size) override;
  140. leveldb::Status RenameFile(const std::string& src,
  141. const std::string& dst) override;
  142. leveldb::Status LockFile(const std::string& fname,
  143. leveldb::FileLock** lock) override;
  144. leveldb::Status UnlockFile(leveldb::FileLock* lock) override;
  145. void Schedule(ScheduleFunc*, void* arg) override;
  146. void StartThread(void (*function)(void* arg), void* arg) override;
  147. leveldb::Status GetTestDirectory(std::string* path) override;
  148. uint64_t NowMicros() override;
  149. void SleepForMicroseconds(int micros) override;
  150. leveldb::Status NewSequentialFile(const std::string& fname,
  151. leveldb::SequentialFile** result) override;
  152. leveldb::Status NewRandomAccessFile(
  153. const std::string& fname,
  154. leveldb::RandomAccessFile** result) override;
  155. leveldb::Status NewWritableFile(const std::string& fname,
  156. leveldb::WritableFile** result) override;
  157. leveldb::Status NewAppendableFile(const std::string& fname,
  158. leveldb::WritableFile** result) override;
  159. leveldb::Status NewLogger(const std::string& fname,
  160. leveldb::Logger** result) override;
  161. void SetReadOnlyFileLimitForTesting(int max_open_files);
  162. protected:
  163. // Constructs a ChromiumEnv instance with a local unrestricted FilesystemProxy
  164. // instance that performs direct filesystem access.
  165. explicit ChromiumEnv(const std::string& name);
  166. // Constructs a ChromiumEnv instance with a custom FilesystemProxy instance.
  167. ChromiumEnv(const std::string& name,
  168. std::unique_ptr<storage::FilesystemProxy> filesystem);
  169. static const char* FileErrorString(base::File::Error error);
  170. private:
  171. void RemoveBackupFiles(const base::FilePath& dir);
  172. // BGThread() is the body of the background thread
  173. void BGThread();
  174. static void BGThreadWrapper(void* arg) {
  175. reinterpret_cast<ChromiumEnv*>(arg)->BGThread();
  176. }
  177. const std::unique_ptr<storage::FilesystemProxy> filesystem_;
  178. base::FilePath test_directory_;
  179. std::string name_;
  180. base::Lock mu_;
  181. base::ConditionVariable bgsignal_;
  182. bool started_bgthread_;
  183. // Entry per Schedule() call
  184. struct BGItem {
  185. void* arg;
  186. void (*function)(void*);
  187. };
  188. using BGQueue = base::circular_deque<BGItem>;
  189. BGQueue queue_;
  190. std::unique_ptr<leveldb::Cache> file_cache_;
  191. };
  192. // Tracks databases open via OpenDatabase() method and exposes them to
  193. // memory-infra. The class is thread safe.
  194. class LEVELDB_EXPORT DBTracker {
  195. public:
  196. enum SharedReadCacheUse : int {
  197. // Use for databases whose access pattern is dictated by browser code.
  198. SharedReadCacheUse_Browser = 0,
  199. // Use for databases whose access pattern is directly influenced by Web
  200. // APIs, like Indexed DB, etc.
  201. SharedReadCacheUse_Web,
  202. SharedReadCacheUse_Unified, // When Web == Browser.
  203. SharedReadCacheUse_InMemory, // Shared by all in-memory databases.
  204. SharedReadCacheUse_NumCacheUses
  205. };
  206. // DBTracker singleton instance.
  207. static DBTracker* GetInstance();
  208. // Returns the memory-infra dump for |tracked_db|. Can be used to attach
  209. // additional info to the database dump, or to properly attribute memory
  210. // usage in memory dump providers that also dump |tracked_db|.
  211. // Note that |tracked_db| should be a live database instance produced by
  212. // OpenDatabase() method or leveldb_env::OpenDB() function.
  213. static base::trace_event::MemoryAllocatorDump* GetOrCreateAllocatorDump(
  214. base::trace_event::ProcessMemoryDump* pmd,
  215. leveldb::DB* tracked_db);
  216. // Returns the memory-infra dump for |tracked_memenv|. Can be used to attach
  217. // additional info to the database dump, or to properly attribute memory
  218. // usage in memory dump providers that also dump |tracked_memenv|.
  219. // Note that |tracked_memenv| should be a live Env instance produced by
  220. // leveldb_chrome::NewMemEnv().
  221. static base::trace_event::MemoryAllocatorDump* GetOrCreateAllocatorDump(
  222. base::trace_event::ProcessMemoryDump* pmd,
  223. leveldb::Env* tracked_memenv);
  224. // Provides extra information about a tracked database.
  225. class TrackedDB : public leveldb::DB {
  226. public:
  227. // Name that OpenDatabase() was called with.
  228. virtual const std::string& name() const = 0;
  229. // Options used when opening the database.
  230. virtual SharedReadCacheUse block_cache_type() const = 0;
  231. };
  232. // Opens a database and starts tracking it. As long as the opened database
  233. // is alive (i.e. its instance is not destroyed) the database is exposed to
  234. // memory-infra and is enumerated by VisitDatabases() method.
  235. // This function is an implementation detail of leveldb_env::OpenDB(), and
  236. // has similar guarantees regarding |dbptr| argument.
  237. leveldb::Status OpenDatabase(const leveldb_env::Options& options,
  238. const std::string& name,
  239. TrackedDB** dbptr);
  240. private:
  241. class MemoryDumpProvider;
  242. class TrackedDBImpl;
  243. using DatabaseVisitor = base::RepeatingCallback<void(TrackedDB*)>;
  244. friend class ChromiumEnvDBTrackerTest;
  245. FRIEND_TEST_ALL_PREFIXES(ChromiumEnvDBTrackerTest, IsTrackedDB);
  246. FRIEND_TEST_ALL_PREFIXES(ChromiumEnvDBTrackerTest, MemoryDumpCreation);
  247. FRIEND_TEST_ALL_PREFIXES(ChromiumEnvDBTrackerTest, MemEnvMemoryDumpCreation);
  248. DBTracker();
  249. ~DBTracker();
  250. // Calls |visitor| for each live database. The database is live from the
  251. // point it was returned from OpenDatabase() and up until its instance is
  252. // destroyed.
  253. // The databases may be visited in an arbitrary order.
  254. // This function takes a lock, preventing any database from being opened or
  255. // destroyed (but doesn't lock the databases themselves).
  256. void VisitDatabases(const DatabaseVisitor& visitor);
  257. // Checks if |db| is tracked.
  258. bool IsTrackedDB(const leveldb::DB* db) const;
  259. void DatabaseOpened(TrackedDBImpl* database, SharedReadCacheUse cache_use);
  260. void DatabaseDestroyed(TrackedDBImpl* database, SharedReadCacheUse cache_use);
  261. // Protect databases_ and mdp_ members.
  262. mutable base::Lock databases_lock_;
  263. base::LinkedList<TrackedDBImpl> databases_;
  264. std::unique_ptr<MemoryDumpProvider> mdp_;
  265. DISALLOW_COPY_AND_ASSIGN(DBTracker);
  266. };
  267. // Opens a database with the specified "name" and "options" (see note) and
  268. // exposes it to Chrome's tracing (see DBTracker for details). The function
  269. // guarantees that:
  270. // 1. |dbptr| is not touched on failure
  271. // 2. |dbptr| is not NULL on success
  272. //
  273. // Note: All |options| values are honored, except if options.env is an in-memory
  274. // Env. In this case the block cache is disabled and a minimum write buffer size
  275. // is used to conserve memory with all other values honored.
  276. LEVELDB_EXPORT leveldb::Status OpenDB(const leveldb_env::Options& options,
  277. const std::string& name,
  278. std::unique_ptr<leveldb::DB>* dbptr);
  279. // Copies the content of |dbptr| into a fresh database to remove traces of
  280. // deleted data. |options| and |name| of the old database are required to create
  281. // an identical copy. |dbptr| will be replaced with the new database on success.
  282. // If the rewrite fails e.g. because we can't write to the temporary location,
  283. // the old db is returned if possible, otherwise |*dbptr| can become NULL.
  284. // The rewrite will only be performed if |kLevelDBRewriteFeature| is enabled.
  285. LEVELDB_EXPORT leveldb::Status RewriteDB(const leveldb_env::Options& options,
  286. const std::string& name,
  287. std::unique_ptr<leveldb::DB>* dbptr);
  288. LEVELDB_EXPORT base::StringPiece MakeStringPiece(const leveldb::Slice& s);
  289. LEVELDB_EXPORT leveldb::Slice MakeSlice(const base::StringPiece& s);
  290. LEVELDB_EXPORT leveldb::Slice MakeSlice(base::span<const uint8_t> s);
  291. } // namespace leveldb_env
  292. #endif // THIRD_PARTY_LEVELDATABASE_ENV_CHROMIUM_H_