service to transfer user keys to a new service/database/rsakey (#137)

dev/.gitignore

@@ -1,7 +1,9 @@
 secrets.json
 database.json
+databaseTo.json
 
 # Development keys
 *.key
 *.crt
 *.pfx
+*.lock

src/KeyConnector/HostedServices/TransferToHostedService.cs

@@ -0,0 +1,121 @@
+using System;
+using System.IO;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+using Bit.KeyConnector.Repositories;
+using Bit.KeyConnector.Services;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+
+namespace Bit.KeyConnector.HostedServices
+{
+    public class TransferToHostedService : IHostedService, IDisposable
+    {
+        private readonly IUserKeyRepository _userKeyRepository;
+        private readonly ICryptoService _cryptoService;
+        private readonly KeyConnectorSettings _keyConnectorSettings;
+        private readonly ILogger<TransferToHostedService> _logger;
+
+        public TransferToHostedService(
+            IUserKeyRepository userKeyRepository,
+            ICryptoService cryptoService,
+            KeyConnectorSettings keyConnectorSettings,
+            ILogger<TransferToHostedService> logger)
+        {
+            _userKeyRepository = userKeyRepository;
+            _cryptoService = cryptoService;
+            _keyConnectorSettings = keyConnectorSettings;
+            _logger = logger;
+        }
+
+        public virtual async Task StartAsync(CancellationToken cancellationToken)
+        {
+            if (!ShouldRun())
+            {
+                return;
+            }
+
+            _logger.LogInformation("Starting service to transfer to new database.");
+            WriteLockfile();
+
+            var transferToProvider = ConfigureTransferToServices();
+            var transferToCryptoService = transferToProvider.GetRequiredService<ICryptoService>();
+            var transferToUserKeyRepository = transferToProvider.GetRequiredService<IUserKeyRepository>();
+
+            var userKeys = await _userKeyRepository.ReadAllAsync();
+            _logger.LogInformation("Found {0} user keys to transfer.", userKeys.Count);
+            foreach (var userKey in userKeys)
+            {
+                try
+                {
+                    var key = await _cryptoService.AesDecryptToB64Async(userKey.Key);
+                    userKey.Key = await transferToCryptoService.AesEncryptToB64Async(key);
+                    await transferToUserKeyRepository.CreateAsync(userKey);
+                }
+                catch (Exception e)
+                {
+                    _logger.LogWarning(e, "Failed to transfer user key {0}. Skip it.", userKey.Id);
+                }
+            }
+            _logger.LogInformation("Finished transferring user keys.");
+        }
+
+        public virtual Task StopAsync(CancellationToken cancellationToken)
+        {
+            return Task.FromResult(0);
+        }
+
+        public virtual void Dispose()
+        { }
+
+        private bool ShouldRun()
+        {
+            if (string.IsNullOrWhiteSpace(_keyConnectorSettings.TransferTo.LockfilePath))
+            {
+                return false;
+            }
+
+            if (File.Exists(_keyConnectorSettings.TransferTo.LockfilePath))
+            {
+                _logger.LogInformation("Not running transfer service due to lock file existing at {0}.",
+                    _keyConnectorSettings.TransferTo.LockfilePath);
+                return false;
+            }
+
+            if (_keyConnectorSettings.TransferTo?.RsaKey == null ||
+                _keyConnectorSettings.TransferTo.Database == null)
+            {
+                _logger.LogInformation("Not running transfer service due to missing settings.");
+                return false;
+            }
+
+            return true;
+        }
+
+        private void WriteLockfile()
+        {
+            using var fs = File.Create(_keyConnectorSettings.TransferTo.LockfilePath);
+            var nowString = Encoding.UTF8.GetBytes(DateTime.UtcNow.ToString());
+            fs.Write(nowString, 0, nowString.Length);
+        }
+
+        private IServiceProvider ConfigureTransferToServices()
+        {
+            var settings = new KeyConnectorSettings
+            {
+                RsaKey = _keyConnectorSettings.TransferTo.RsaKey,
+                Certificate = _keyConnectorSettings.TransferTo.Certificate,
+                Database = _keyConnectorSettings.TransferTo.Database
+            };
+            var transferToServices = new ServiceCollection();
+            transferToServices.AddSingleton(s => settings);
+            transferToServices.AddRsaKeyProvider(settings);
+            transferToServices.AddBaseServices();
+            var efDatabaseProvider = transferToServices.AddDatabase(settings);
+            var transferToProvider = transferToServices.BuildServiceProvider();
+            return transferToProvider;
+        }
+    }
+}

src/KeyConnector/KeyConnectorSettings.cs

@@ -9,6 +9,8 @@
         public CertificateSettings Certificate { get; set; }
         public RsaKeySettings RsaKey { get; set; }
 
+        public TransferToSettings TransferTo { get; set; }
+
         public class CertificateSettings
         {
             public string Provider { get; set; }
@@ -88,5 +90,13 @@
             public string MongoConnectionString { get; set; }
             public string MongoDatabaseName { get; set; }
         }
+
+        public class TransferToSettings
+        {
+            public string LockfilePath { get; set; }
+            public DatabaseSettings Database { get; set; }
+            public CertificateSettings Certificate { get; set; }
+            public RsaKeySettings RsaKey { get; set; }
+        }
     }
 }

src/KeyConnector/Properties/launchSettings.json

@@ -15,7 +15,7 @@
         "ASPNETCORE_ENVIRONMENT": "Development"
       }
     },
-    "SsoAgent": {
+    "KeyConnector": {
       "commandName": "Project",
       "dotnetRunMessages": "true",
       "launchBrowser": false,

src/KeyConnector/Repositories/EntityFramework/UserKeyRepository.cs

@@ -1,6 +1,9 @@
 using System;
+using System.Collections.Generic;
+using System.Linq;
 using System.Threading.Tasks;
 using Bit.KeyConnector.Models;
+using Microsoft.EntityFrameworkCore;
 using Microsoft.Extensions.DependencyInjection;
 
 namespace Bit.KeyConnector.Repositories.EntityFramework
@@ -28,6 +31,14 @@ namespace Bit.KeyConnector.Repositories.EntityFramework
             return entity?.ToUserKeyModel();
         }
 
+        public virtual async Task<List<UserKeyModel>> ReadAllAsync()
+        {
+            using var scope = ServiceScopeFactory.CreateScope();
+            var dbContext = GetDatabaseContext(scope);
+            var entities = await dbContext.UserKeys.ToListAsync();
+            return entities.Select(e => e.ToUserKeyModel()).ToList();
+        }
+
         public virtual async Task UpdateAsync(UserKeyModel item)
         {
             using var scope = ServiceScopeFactory.CreateScope();

src/KeyConnector/Repositories/IUserKeyRepository.cs

@@ -1,9 +1,12 @@
 using System;
+using System.Collections.Generic;
+using System.Threading.Tasks;
 using Bit.KeyConnector.Models;
 
 namespace Bit.KeyConnector.Repositories
 {
     public interface IUserKeyRepository : IRepository<UserKeyModel, Guid>
     {
+        Task<List<UserKeyModel>> ReadAllAsync();
     }
 }

src/KeyConnector/Repositories/JsonFile/UserKeyRepository.cs

@@ -1,4 +1,6 @@
 using System;
+using System.Collections.Generic;
+using System.Linq;
 using System.Threading.Tasks;
 using Bit.KeyConnector.Models;
 using JsonFlatFileDataStore;
@@ -11,6 +13,13 @@ namespace Bit.KeyConnector.Repositories.JsonFile
         : base(dataStore, "userKey")
         { }
 
+        public virtual Task<List<UserKeyModel>> ReadAllAsync()
+        {
+            var collection = DataStore.GetCollection<UserKeyModel>(CollectionName);
+            var keys = collection.AsQueryable().ToList();
+            return Task.FromResult(keys);
+        }
+
         public override async Task CreateAsync(UserKeyModel item)
         {
             var collection = DataStore.GetCollection<JsonUserKeyModel>(CollectionName);

src/KeyConnector/Repositories/Mongo/UserKeyRepository.cs

@@ -1,6 +1,9 @@
 using System;
+using System.Collections.Generic;
+using System.Collections;
 using System.Threading.Tasks;
 using Bit.KeyConnector.Models;
+using JsonFlatFileDataStore;
 using MongoDB.Driver;
 
 namespace Bit.KeyConnector.Repositories.Mongo
@@ -21,6 +24,11 @@ namespace Bit.KeyConnector.Repositories.Mongo
             return Collection.Find(d => d.Id == id).FirstOrDefaultAsync();
         }
 
+        public virtual Task<List<UserKeyModel>> ReadAllAsync()
+        {
+            return Collection.Find(d => true).ToListAsync();
+        }
+
         public virtual async Task UpdateAsync(UserKeyModel item)
         {
             await Collection.ReplaceOneAsync(d => d.Id == item.Id, item);

src/KeyConnector/ServiceCollectionExtensions.cs

@@ -0,0 +1,181 @@
+using System;
+using System.Security.Claims;
+using Bit.KeyConnector.Repositories;
+using Bit.KeyConnector.Services;
+using IdentityModel;
+using IdentityServer4.AccessTokenValidation;
+using JsonFlatFileDataStore;
+using Microsoft.AspNetCore.Builder;
+using Microsoft.AspNetCore.Hosting;
+using Microsoft.AspNetCore.Identity;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
+
+namespace Bit.KeyConnector
+{
+    public static class ServiceCollectionExtensions
+    {
+        public static void AddRsaKeyProvider(
+            this IServiceCollection services,
+            KeyConnectorSettings settings)
+        {
+            var rsaKeyProvider = settings.RsaKey.Provider?.ToLowerInvariant();
+            if (rsaKeyProvider == "certificate" || rsaKeyProvider == "pkcs11")
+            {
+                if (rsaKeyProvider == "certificate")
+                {
+                    services.AddSingleton<IRsaKeyService, LocalCertificateRsaKeyService>();
+                }
+                else if (rsaKeyProvider == "pkcs11")
+                {
+                    services.AddSingleton<IRsaKeyService, Pkcs11RsaKeyService>();
+                }
+
+                var certificateProvider = settings.Certificate.Provider?.ToLowerInvariant();
+                if (certificateProvider == "store")
+                {
+                    services.AddSingleton<ICertificateProviderService, StoreCertificateProviderService>();
+                }
+                else if (certificateProvider == "filesystem")
+                {
+                    services.AddSingleton<ICertificateProviderService, FilesystemCertificateProviderService>();
+                }
+                else if (certificateProvider == "azurestorage")
+                {
+                    services.AddSingleton<ICertificateProviderService, AzureStorageCertificateProviderService>();
+                }
+                else if (certificateProvider == "azurekv")
+                {
+                    services.AddSingleton<ICertificateProviderService, AzureKeyVaultCertificateProviderService>();
+                }
+                else if (certificateProvider == "vault")
+                {
+                    services.AddSingleton<ICertificateProviderService, HashicorpVaultCertificateProviderService>();
+                }
+                else
+                {
+                    throw new Exception("Unknown certificate provider configured.");
+                }
+            }
+            else if (rsaKeyProvider == "azurekv")
+            {
+                services.AddSingleton<IRsaKeyService, AzureKeyVaultRsaKeyService>();
+            }
+            else if (rsaKeyProvider == "gcpkms")
+            {
+                services.AddSingleton<IRsaKeyService, GoogleCloudKmsRsaKeyService>();
+            }
+            else if (rsaKeyProvider == "awskms")
+            {
+                services.AddSingleton<IRsaKeyService, AwsKmsRsaKeyService>();
+            }
+            else
+            {
+                throw new Exception("Unknown rsa key provider configured.");
+            }
+        }
+
+        public static bool AddDatabase(
+            this IServiceCollection services,
+            KeyConnectorSettings settings)
+        {
+            var databaseProvider = settings.Database.Provider?.ToLowerInvariant();
+            var efDatabaseProvider = databaseProvider == "sqlserver" || databaseProvider == "postgresql" ||
+                databaseProvider == "mysql" || databaseProvider == "sqlite";
+
+            if (databaseProvider == "json")
+            {
+                // Assign foobar to keyProperty in order to not use incrementing Id functionality
+                services.AddSingleton<IDataStore>(
+                    new DataStore(settings.Database.JsonFilePath, keyProperty: "--foobar--"));
+                services.AddSingleton<IApplicationDataRepository, Repositories.JsonFile.ApplicationDataRepository>();
+                services.AddSingleton<IUserKeyRepository, Repositories.JsonFile.UserKeyRepository>();
+            }
+            else if (databaseProvider == "mongo")
+            {
+                services.AddSingleton<IApplicationDataRepository, Repositories.Mongo.ApplicationDataRepository>();
+                services.AddSingleton<IUserKeyRepository, Repositories.Mongo.UserKeyRepository>();
+            }
+            else if (efDatabaseProvider)
+            {
+                if (databaseProvider == "sqlserver")
+                {
+                    services.AddDbContext<Repositories.EntityFramework.DatabaseContext,
+                        Repositories.EntityFramework.SqlServerDatabaseContext>();
+                }
+                else if (databaseProvider == "postgresql")
+                {
+                    services.AddDbContext<Repositories.EntityFramework.DatabaseContext,
+                        Repositories.EntityFramework.PostgreSqlDatabaseContext>();
+                }
+                else if (databaseProvider == "mysql")
+                {
+                    services.AddDbContext<Repositories.EntityFramework.DatabaseContext,
+                        Repositories.EntityFramework.MySqlDatabaseContext>();
+                }
+                else if (databaseProvider == "sqlite")
+                {
+                    services.AddDbContext<Repositories.EntityFramework.DatabaseContext,
+                        Repositories.EntityFramework.SqliteDatabaseContext>();
+                }
+                services.AddSingleton<IApplicationDataRepository,
+                    Repositories.EntityFramework.ApplicationDataRepository>();
+                services.AddSingleton<IUserKeyRepository, Repositories.EntityFramework.UserKeyRepository>();
+            }
+            else
+            {
+                throw new Exception("No database configured.");
+            }
+
+            return efDatabaseProvider;
+        }
+
+        public static void AddAuthentication(
+            this IServiceCollection services,
+            IWebHostEnvironment environment,
+            KeyConnectorSettings settings)
+        {
+            services.Configure<IdentityOptions>(options =>
+            {
+                options.ClaimsIdentity = new ClaimsIdentityOptions
+                {
+                    UserNameClaimType = JwtClaimTypes.Email,
+                    UserIdClaimType = JwtClaimTypes.Subject
+                };
+            });
+            services
+                .AddAuthentication(IdentityServerAuthenticationDefaults.AuthenticationScheme)
+                .AddIdentityServerAuthentication(options =>
+                {
+                    options.Authority = settings.IdentityServerUri;
+                    options.RequireHttpsMetadata = !environment.IsDevelopment() &&
+                        settings.IdentityServerUri.StartsWith("https");
+                    options.NameClaimType = ClaimTypes.Email;
+                    options.SupportedTokens = SupportedTokens.Jwt;
+                });
+
+            services
+                .AddAuthorization(config =>
+                {
+                    config.AddPolicy("Application", policy =>
+                    {
+                        policy.RequireAuthenticatedUser();
+                        policy.RequireClaim(JwtClaimTypes.AuthenticationMethod, "Application", "external");
+                        policy.RequireClaim(JwtClaimTypes.Scope, "api");
+                    });
+                });
+
+            if (environment.IsDevelopment())
+            {
+                Microsoft.IdentityModel.Logging.IdentityModelEventSource.ShowPII = true;
+            }
+        }
+
+        public static void AddBaseServices(
+            this IServiceCollection services)
+        {
+            services.AddSingleton<ICryptoFunctionService, CryptoFunctionService>();
+            services.AddSingleton<ICryptoService, CryptoService>();
+        }
+    }
+}

src/KeyConnector/Startup.cs

@@ -1,18 +1,10 @@
-using System;
+using System.Globalization;
-using System.Globalization;
-using System.Security.Claims;
-using Bit.KeyConnector.Repositories;
 using Bit.KeyConnector.Services;
-using IdentityModel;
-using IdentityServer4.AccessTokenValidation;
-using JsonFlatFileDataStore;
 using Microsoft.AspNetCore.Builder;
 using Microsoft.AspNetCore.Hosting;
-using Microsoft.AspNetCore.Identity;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
-using Microsoft.Extensions.Diagnostics.HealthChecks;
 using Serilog;
 
 namespace Bit.KeyConnector
@@ -39,16 +31,14 @@ namespace Bit.KeyConnector
             ConfigurationBinder.Bind(Configuration.GetSection("KeyConnectorSettings"), settings);
             services.AddSingleton(s => settings);
 
-            AddAuthentication(services, settings);
+            services.AddAuthentication(Environment, settings);
-            AddRsaKeyProvider(services, settings);
+            services.AddRsaKeyProvider(settings);
-
+            services.AddBaseServices();
-            services.AddSingleton<ICryptoFunctionService, CryptoFunctionService>();
+            var efDatabaseProvider = services.AddDatabase(settings);
-            services.AddSingleton<ICryptoService, CryptoService>();
-
-            var efDatabaseProvider = AddDatabase(services, settings);
 
             services.AddControllers();
 
+            services.AddHostedService<HostedServices.TransferToHostedService>();
             if (efDatabaseProvider)
             {
                 services.AddHostedService<HostedServices.DatabaseMigrationHostedService>();
@@ -86,154 +76,5 @@ namespace Bit.KeyConnector
                 }
             });
         }
-
-        private bool AddDatabase(IServiceCollection services, KeyConnectorSettings settings)
-        {
-            var databaseProvider = settings.Database.Provider?.ToLowerInvariant();
-            var efDatabaseProvider = databaseProvider == "sqlserver" || databaseProvider == "postgresql" ||
-                databaseProvider == "mysql" || databaseProvider == "sqlite";
-
-            if (databaseProvider == "json")
-            {
-                // Assign foobar to keyProperty in order to not use incrementing Id functionality
-                services.AddSingleton<IDataStore>(
-                    new DataStore(settings.Database.JsonFilePath, keyProperty: "--foobar--"));
-                services.AddSingleton<IApplicationDataRepository, Repositories.JsonFile.ApplicationDataRepository>();
-                services.AddSingleton<IUserKeyRepository, Repositories.JsonFile.UserKeyRepository>();
-            }
-            else if (databaseProvider == "mongo")
-            {
-                services.AddSingleton<IApplicationDataRepository, Repositories.Mongo.ApplicationDataRepository>();
-                services.AddSingleton<IUserKeyRepository, Repositories.Mongo.UserKeyRepository>();
-            }
-            else if (efDatabaseProvider)
-            {
-                if (databaseProvider == "sqlserver")
-                {
-                    services.AddDbContext<Repositories.EntityFramework.DatabaseContext,
-                        Repositories.EntityFramework.SqlServerDatabaseContext>();
-                }
-                else if (databaseProvider == "postgresql")
-                {
-                    services.AddDbContext<Repositories.EntityFramework.DatabaseContext,
-                        Repositories.EntityFramework.PostgreSqlDatabaseContext>();
-                }
-                else if (databaseProvider == "mysql")
-                {
-                    services.AddDbContext<Repositories.EntityFramework.DatabaseContext,
-                        Repositories.EntityFramework.MySqlDatabaseContext>();
-                }
-                else if (databaseProvider == "sqlite")
-                {
-                    services.AddDbContext<Repositories.EntityFramework.DatabaseContext,
-                        Repositories.EntityFramework.SqliteDatabaseContext>();
-                }
-                services.AddSingleton<IApplicationDataRepository,
-                    Repositories.EntityFramework.ApplicationDataRepository>();
-                services.AddSingleton<IUserKeyRepository, Repositories.EntityFramework.UserKeyRepository>();
-            }
-            else
-            {
-                throw new Exception("No database configured.");
-            }
-
-            return efDatabaseProvider;
-        }
-
-        private void AddRsaKeyProvider(IServiceCollection services, KeyConnectorSettings settings)
-        {
-            var rsaKeyProvider = settings.RsaKey.Provider?.ToLowerInvariant();
-            if (rsaKeyProvider == "certificate" || rsaKeyProvider == "pkcs11")
-            {
-                if (rsaKeyProvider == "certificate")
-                {
-                    services.AddSingleton<IRsaKeyService, LocalCertificateRsaKeyService>();
-                }
-                else if (rsaKeyProvider == "pkcs11")
-                {
-                    services.AddSingleton<IRsaKeyService, Pkcs11RsaKeyService>();
-                }
-
-                var certificateProvider = settings.Certificate.Provider?.ToLowerInvariant();
-                if (certificateProvider == "store")
-                {
-                    services.AddSingleton<ICertificateProviderService, StoreCertificateProviderService>();
-                }
-                else if (certificateProvider == "filesystem")
-                {
-                    services.AddSingleton<ICertificateProviderService, FilesystemCertificateProviderService>();
-                }
-                else if (certificateProvider == "azurestorage")
-                {
-                    services.AddSingleton<ICertificateProviderService, AzureStorageCertificateProviderService>();
-                }
-                else if (certificateProvider == "azurekv")
-                {
-                    services.AddSingleton<ICertificateProviderService, AzureKeyVaultCertificateProviderService>();
-                }
-                else if (certificateProvider == "vault")
-                {
-                    services.AddSingleton<ICertificateProviderService, HashicorpVaultCertificateProviderService>();
-                }
-                else
-                {
-                    throw new Exception("Unknown certificate provider configured.");
-                }
-            }
-            else if (rsaKeyProvider == "azurekv")
-            {
-                services.AddSingleton<IRsaKeyService, AzureKeyVaultRsaKeyService>();
-            }
-            else if (rsaKeyProvider == "gcpkms")
-            {
-                services.AddSingleton<IRsaKeyService, GoogleCloudKmsRsaKeyService>();
-            }
-            else if (rsaKeyProvider == "awskms")
-            {
-                services.AddSingleton<IRsaKeyService, AwsKmsRsaKeyService>();
-            }
-            else
-            {
-                throw new Exception("Unknown rsa key provider configured.");
-            }
-        }
-
-        private void AddAuthentication(IServiceCollection services, KeyConnectorSettings settings)
-        {
-            services.Configure<IdentityOptions>(options =>
-            {
-                options.ClaimsIdentity = new ClaimsIdentityOptions
-                {
-                    UserNameClaimType = JwtClaimTypes.Email,
-                    UserIdClaimType = JwtClaimTypes.Subject
-                };
-            });
-            services
-                .AddAuthentication(IdentityServerAuthenticationDefaults.AuthenticationScheme)
-                .AddIdentityServerAuthentication(options =>
-                {
-                    options.Authority = settings.IdentityServerUri;
-                    options.RequireHttpsMetadata = !Environment.IsDevelopment() &&
-                        settings.IdentityServerUri.StartsWith("https");
-                    options.NameClaimType = ClaimTypes.Email;
-                    options.SupportedTokens = SupportedTokens.Jwt;
-                });
-
-            services
-                .AddAuthorization(config =>
-                {
-                    config.AddPolicy("Application", policy =>
-                    {
-                        policy.RequireAuthenticatedUser();
-                        policy.RequireClaim(JwtClaimTypes.AuthenticationMethod, "Application", "external");
-                        policy.RequireClaim(JwtClaimTypes.Scope, "api");
-                    });
-                });
-
-            if (Environment.IsDevelopment())
-            {
-                Microsoft.IdentityModel.Logging.IdentityModelEventSource.ShowPII = true;
-            }
-        }
     }
 }

src/KeyConnector/appsettings.json

@@ -36,6 +36,9 @@
     "certificate": {
       "provider": "filesystem",
       "filesystemPath": "/etc/bitwarden/key.pfx"
+    },
+    "transferTo": {
+      "lockfilePath": "/etc/bitwarden/transfer.lock"
     }
   }
 }