Added cipher service with bulk import to account controller

src/Api/Controllers/AccountsController.cs

@@ -11,6 +11,7 @@ using Bit.Core.Domains;
 using Bit.Core.Enums;
 using Bit.Core;
 using System.Security.Claims;
+using System.Linq;
 
 namespace Bit.Api.Controllers
 {
@@ -19,16 +20,18 @@ namespace Bit.Api.Controllers
     public class AccountsController : Controller
     {
         private readonly IUserService _userService;
+        private readonly ICipherService _cipherService;
         private readonly UserManager<User> _userManager;
         private readonly CurrentContext _currentContext;
 
         public AccountsController(
-            IDataProtectionProvider dataProtectionProvider,
             IUserService userService,
+            ICipherService cipherService,
             UserManager<User> userManager,
             CurrentContext currentContext)
         {
             _userService = userService;
+            _cipherService = cipherService;
             _userManager = userManager;
             _currentContext = currentContext;
         }
@@ -206,5 +209,14 @@ namespace Bit.Api.Controllers
             var response = new TwoFactorResponseModel(user);
             return response;
         }
+
+        [HttpPost("import")]
+        public async Task PostImport([FromBody]ImportRequestModel model)
+        {
+            await _cipherService.ImportCiphersAsync(
+                model.Folders.Select(f => f.ToFolder(User.GetUserId())).ToList(),
+                model.Sites.Select(s => s.ToSite(User.GetUserId())).ToList(),
+                model.SiteRelationships);
+        }
     }
 }

src/Api/Controllers/SitesController.cs

@@ -144,5 +144,4 @@ namespace Bit.Api.Controllers
             }
         }
     }
-
 }

src/Api/Models/Request/Accounts/ImportRequestModel.cs

@@ -0,0 +1,12 @@
+using System;
+using System.Collections.Generic;
+
+namespace Bit.Api.Models
+{
+    public class ImportRequestModel
+    {
+        public FolderRequestModel[] Folders { get; set; }
+        public SiteRequestModel[] Sites { get; set; }
+        public KeyValuePair<int, int>[] SiteRelationships { get; set; }
+    }
+}

src/Api/Models/Request/Ciphers/CipherRequestModel.cs

@@ -71,10 +71,6 @@ namespace Bit.Api.Models
                 {
                     yield return new ValidationResult("Uri is required for a site cypher.", new[] { "Uri" });
                 }
-                if(string.IsNullOrWhiteSpace(Username))
-                {
-                    yield return new ValidationResult("Username is required for a site cypher.", new[] { "Username" });
-                }
                 if(string.IsNullOrWhiteSpace(Password))
                 {
                     yield return new ValidationResult("Password is required for a site cypher.", new[] { "Password" });

src/Api/Models/Request/Sites/SiteRequestModel.cs

@@ -14,7 +14,6 @@ namespace Bit.Api.Models
         [Required]
         [EncryptedString]
         public string Uri { get; set; }
-        [Required]
         [EncryptedString]
         public string Username { get; set; }
         [Required]
@@ -31,7 +30,7 @@ namespace Bit.Api.Models
                 FolderId = string.IsNullOrWhiteSpace(FolderId) ? null : FolderId,
                 Name = Name,
                 Uri = Uri,
-                Username = Username,
+                Username = string.IsNullOrWhiteSpace(Username) ? null : Username,
                 Password = Password,
                 Notes = string.IsNullOrWhiteSpace(Notes) ? null : Notes
             };
@@ -42,7 +41,7 @@ namespace Bit.Api.Models
             existingSite.FolderId = string.IsNullOrWhiteSpace(FolderId) ? null : FolderId;
             existingSite.Name = Name;
             existingSite.Uri = Uri;
-            existingSite.Username = Username;
+            existingSite.Username = string.IsNullOrWhiteSpace(Username) ? null : Username;
             existingSite.Password = Password;
             existingSite.Notes = string.IsNullOrWhiteSpace(Notes) ? null : Notes;
 

src/Api/Startup.cs

@@ -53,7 +53,7 @@ namespace Bit.Api
             services.AddSingleton(s => globalSettings);
 
             // Repositories
-            var documentDBClient = DocumentClientHelpers.InitClient(globalSettings.DocumentDB);
+            var documentDBClient = DocumentDBHelpers.InitClient(globalSettings.DocumentDB);
             services.AddSingleton<IUserRepository>(s => new Repos.UserRepository(documentDBClient, globalSettings.DocumentDB.DatabaseId));
             services.AddSingleton<ISiteRepository>(s => new Repos.SiteRepository(documentDBClient, globalSettings.DocumentDB.DatabaseId));
             services.AddSingleton<IFolderRepository>(s => new Repos.FolderRepository(documentDBClient, globalSettings.DocumentDB.DatabaseId));
@@ -116,6 +116,7 @@ namespace Bit.Api
 
             // Services
             services.AddSingleton<IMailService, MailService>();
+            services.AddSingleton<ICipherService, CipherService>();
             services.AddScoped<IUserService, UserService>();
 
             // Cors

src/Core/Repositories/DocumentDB/CipherRepository.cs

@@ -4,6 +4,7 @@ using System.Linq;
 using System.Threading.Tasks;
 using Microsoft.Azure.Documents.Client;
 using Bit.Core.Domains;
+using Bit.Core.Repositories.DocumentDB.Utilities;
 
 namespace Bit.Core.Repositories.DocumentDB
 {
@@ -15,26 +16,53 @@ namespace Bit.Core.Repositories.DocumentDB
 
         public async Task UpdateDirtyCiphersAsync(IEnumerable<dynamic> ciphers)
         {
-            // Make sure we are dealing with cipher types since we accept any via dynamic.
-            var cleanedCiphers = ciphers.Where(c => c is Cipher);
-            if(cleanedCiphers.Count() == 0)
+            await DocumentDBHelpers.QueryWithRetryAsync(async () =>
             {
-                return;
-            }
+                // Make sure we are dealing with cipher types since we accept any via dynamic.
+                var cleanedCiphers = ciphers.Where(c => c is Cipher);
+                if(cleanedCiphers.Count() == 0)
+                {
+                    return;
+                }
 
-            var userId = ((Cipher)cleanedCiphers.First()).UserId;
-            StoredProcedureResponse<int> sprocResponse = await Client.ExecuteStoredProcedureAsync<int>(
-                ResolveSprocIdLink(userId, "bulkUpdateDirtyCiphers"),
-                // Do sets of 50. Recursion will handle the rest below.
-                cleanedCiphers.Take(50),
-                userId,
-                Cipher.TypeValue);
+                var userId = ((Cipher)cleanedCiphers.First()).UserId;
+                StoredProcedureResponse<int> sprocResponse = await Client.ExecuteStoredProcedureAsync<int>(
+                    ResolveSprocIdLink(userId, "bulkUpdateDirtyCiphers"),
+                    // Do sets of 50. Recursion will handle the rest below.
+                    cleanedCiphers.Take(50),
+                    userId);
 
-            var replacedCount = sprocResponse.Response;
-            if(replacedCount != cleanedCiphers.Count())
+                var replacedCount = sprocResponse.Response;
+                if(replacedCount != cleanedCiphers.Count())
+                {
+                    await UpdateDirtyCiphersAsync(cleanedCiphers.Skip(replacedCount));
+                }
+            });
+        }
+
+        public async Task CreateAsync(IEnumerable<dynamic> ciphers)
+        {
+            await DocumentDBHelpers.QueryWithRetryAsync(async () =>
             {
-                await UpdateDirtyCiphersAsync(cleanedCiphers.Skip(replacedCount));
-            }
+                // Make sure we are dealing with cipher types since we accept any via dynamic.
+                var cleanedCiphers = ciphers.Where(c => c is Cipher);
+                if(cleanedCiphers.Count() == 0)
+                {
+                    return;
+                }
+
+                var userId = ((Cipher)cleanedCiphers.First()).UserId;
+                StoredProcedureResponse<int> sprocResponse = await Client.ExecuteStoredProcedureAsync<int>(
+                    ResolveSprocIdLink(userId, "bulkCreate"),
+                    // Do sets of 50. Recursion will handle the rest below.
+                    cleanedCiphers.Take(50));
+
+                var createdCount = sprocResponse.Response;
+                if(createdCount != cleanedCiphers.Count())
+                {
+                    await CreateAsync(cleanedCiphers.Skip(createdCount));
+                }
+            });
         }
     }
 }

src/Core/Repositories/DocumentDB/Stored Procedures/bulkCreate.js

@@ -0,0 +1,60 @@
+/**
+* This script called as stored procedure to import lots of documents in one batch.
+* The script sets response body to the number of docs imported and is called multiple times 
+* by the client until total number of docs desired by the client is imported.
+* @param  {Object[]} docs - Array of documents to import.
+*/
+
+function bulkCreate(docs) {
+    var collection = getContext().getCollection();
+    var collectionLink = collection.getSelfLink();
+
+    // The count of imported docs, also used as current doc index.
+    var count = 0;
+
+    // Validate input.
+    if (!docs) throw new Error('The array is undefined or null.');
+
+    var docsLength = docs.length;
+    if (docsLength == 0) {
+        getContext().getResponse().setBody(0);
+        return;
+    }
+
+    // Call the CRUD API to create a document.
+    tryCreate(docs[count], callback);
+
+    // Note that there are 2 exit conditions:
+    // 1) The createDocument request was not accepted. 
+    //    In this case the callback will not be called, we just call setBody and we are done.
+    // 2) The callback was called docs.length times.
+    //    In this case all documents were created and we don't need to call tryCreate anymore. Just call setBody and we are done.
+    function tryCreate(doc, callback) {
+        var isAccepted = collection.createDocument(collectionLink, doc, callback);
+
+        // If the request was accepted, callback will be called.
+        // Otherwise report current count back to the client, 
+        // which will call the script again with remaining set of docs.
+        // This condition will happen when this stored procedure has been running too long
+        // and is about to get cancelled by the server. This will allow the calling client
+        // to resume this batch from the point we got to before isAccepted was set to false
+        if (!isAccepted) getContext().getResponse().setBody(count);
+    }
+
+    // This is called when collection.createDocument is done and the document has been persisted.
+    function callback(err, doc, options) {
+        if (err) throw err;
+
+        // One more document has been inserted, increment the count.
+        count++;
+
+        if (count >= docsLength) {
+            // If we have created all documents, we are done. Just set the response.
+            getContext().getResponse().setBody(count);
+        }
+        else {
+            // Create next document.
+            tryCreate(docs[count], callback);
+        }
+    }
+}

src/Core/Repositories/DocumentDB/Stored Procedures/bulkUpdateDirtyCiphers.js

@@ -10,7 +10,7 @@ function bulkUpdateDirtyCiphers(ciphers, userId) {
 
     // Validate input.
     if (!ciphers) {
-        throw new Error("The ciphers array is undefined or null.");
+        throw new Error('The ciphers array is undefined or null.');
     }
 
     var ciphersLength = ciphers.length;

src/Core/Repositories/DocumentDB/UserRepository.cs

@@ -1,6 +1,8 @@
 using System;
 using System.Linq;
 using System.Threading.Tasks;
+using Bit.Core.Repositories.DocumentDB.Utilities;
+using Microsoft.Azure.Documents;
 using Microsoft.Azure.Documents.Client;
 
 namespace Bit.Core.Repositories.DocumentDB
@@ -26,7 +28,10 @@ namespace Bit.Core.Repositories.DocumentDB
 
         public async Task ReplaceAndDirtyCiphersAsync(Domains.User user)
         {
-            await Client.ExecuteStoredProcedureAsync<Domains.User>(ResolveSprocIdLink(user, "replaceUserAndDirtyCiphers"), user);
+            await DocumentDBHelpers.QueryWithRetryAsync(async () =>
+            {
+                await Client.ExecuteStoredProcedureAsync<Domains.User>(ResolveSprocIdLink(user, "replaceUserAndDirtyCiphers"), user);
+            });
         }
 
         public override async Task DeleteByIdAsync(string id)

src/Core/Repositories/DocumentDB/Utilities/DocumentDBHelpers.cs

@@ -1,9 +1,11 @@
 using System;
+using System.Threading.Tasks;
+using Microsoft.Azure.Documents;
 using Microsoft.Azure.Documents.Client;
 
 namespace Bit.Core.Repositories.DocumentDB.Utilities
 {
-    public class DocumentClientHelpers
+    public class DocumentDBHelpers
     {
         public static DocumentClient InitClient(GlobalSettings.DocumentDBSettings settings)
         {
@@ -29,6 +31,43 @@ namespace Bit.Core.Repositories.DocumentDB.Utilities
             return client;
         }
 
+        public static async Task QueryWithRetryAsync(Func<Task> func)
+        {
+            var queryComplete = false;
+            while(!queryComplete)
+            {
+                try
+                {
+                    await func();
+                    queryComplete = true;
+                }
+                catch(DocumentClientException e)
+                {
+                    await HandleDocumentClientExceptionAsync(e);
+                }
+                catch(AggregateException e)
+                {
+                    var docEx = e.InnerException as DocumentClientException;
+                    if(docEx != null)
+                    {
+                        await HandleDocumentClientExceptionAsync(docEx);
+                    }
+                }
+            }
+        }
+
+        private static async Task HandleDocumentClientExceptionAsync(DocumentClientException e)
+        {
+            var statusCode = (int)e.StatusCode;
+            if(statusCode == 429 || statusCode == 503)
+            {
+                await Task.Delay(e.RetryAfter);
+            }
+            else {
+                throw e;
+            }
+        }
+
         private static Func<object, string> GetPartitionKeyExtractor()
         {
             return doc =>

src/Core/Repositories/ICipherRepository.cs

@@ -6,5 +6,6 @@ namespace Bit.Core.Repositories
     public interface ICipherRepository
     {
         Task UpdateDirtyCiphersAsync(IEnumerable<dynamic> ciphers);
+        Task CreateAsync(IEnumerable<dynamic> ciphers);
     }
 }

src/Core/Services/CipherService.cs

@@ -0,0 +1,54 @@
+using System;
+using System.Linq;
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using Bit.Core.Domains;
+using Bit.Core.Repositories;
+
+namespace Bit.Core.Services
+{
+    public class CipherService : ICipherService
+    {
+        private readonly IFolderRepository _folderRepository;
+        private readonly ICipherRepository _cipherRepository;
+
+        public CipherService(
+            IFolderRepository folderRepository,
+            ICipherRepository cipherRepository)
+        {
+            _folderRepository = folderRepository;
+            _cipherRepository = cipherRepository;
+        }
+
+        public async Task ImportCiphersAsync(
+            List<Folder> folders,
+            List<Site> sites,
+            IEnumerable<KeyValuePair<int, int>> siteRelationships)
+        {
+            // create all the folders
+            var folderTasks = new List<Task>();
+            foreach(var folder in folders)
+            {
+                folderTasks.Add(_folderRepository.CreateAsync(folder));
+            }
+            await Task.WhenAll(folderTasks);
+
+            // associate the newly created folders to the sites
+            foreach(var relationship in siteRelationships)
+            {
+                var site = sites.ElementAtOrDefault(relationship.Key);
+                var folder = folders.ElementAtOrDefault(relationship.Value);
+
+                if(site == null || folder == null)
+                {
+                    continue;
+                }
+
+                site.FolderId = folder.Id;
+            }
+
+            // create all the sites
+            await _cipherRepository.CreateAsync(sites);
+        }
+    }
+}

src/Core/Services/ICipherService.cs

@@ -0,0 +1,11 @@
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using Bit.Core.Domains;
+
+namespace Bit.Core.Services
+{
+    public interface ICipherService
+    {
+        Task ImportCiphersAsync(List<Folder> folders, List<Site> sites, IEnumerable<KeyValuePair<int, int>> siteRelationships);
+    }
+}