diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 00000000..7a2f97d6 --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1,2 @@ +# Scala Steward: Reformat with scalafmt 2.7.5 +c549b943c5d68c27786a7aa1bbe70aa4f23e1710 diff --git a/.scalafmt.conf b/.scalafmt.conf index 3c63e62f..243c541a 100644 --- a/.scalafmt.conf +++ b/.scalafmt.conf @@ -1,4 +1,4 @@ -version = "2.0.1" +version = "2.7.5" style = defaultWithAlign maxColumn = 150 rewriteTokens = { diff --git a/build.sbt b/build.sbt index 3bd65a5c..bc019311 100644 --- a/build.sbt +++ b/build.sbt @@ -81,7 +81,10 @@ lazy val dockerSettings = Seq( dockerCommands := { val cmds = dockerCommands.value val injected = Seq( - Cmd("RUN", "apt update && apt install -y fuse libfuse2 libfuse-dev python3-pip && rm -rf /var/lib/apt/lists/*"), // TODO https://github.com/docker/for-mac/issues/3431 + Cmd( + "RUN", + "apt update && apt install -y fuse libfuse2 libfuse-dev python3-pip && rm -rf /var/lib/apt/lists/*" + ), // TODO https://github.com/docker/for-mac/issues/3431 Cmd("RUN", "echo 'user_allow_other' >> /etc/fuse.conf"), Cmd( "RUN", diff --git a/cache/larray/src/main/scala/com/karasiq/cache/larray/LArrayChunkCache.scala b/cache/larray/src/main/scala/com/karasiq/cache/larray/LArrayChunkCache.scala index 282bbacc..797e9668 100644 --- a/cache/larray/src/main/scala/com/karasiq/cache/larray/LArrayChunkCache.scala +++ b/cache/larray/src/main/scala/com/karasiq/cache/larray/LArrayChunkCache.scala @@ -7,7 +7,7 @@ import com.karasiq.shadowcloud.cache.ChunkCache import com.karasiq.shadowcloud.model.Chunk private[cache] class LArrayChunkCache(size: Int)(implicit ec: ExecutionContext) extends ChunkCache { - protected val cache = LArrayAsyncLRUCache[Chunk](size) + protected val cache = LArrayAsyncLRUCache[Chunk](size) protected var lastChunk: Chunk = _ def readCached(chunk: Chunk, getChunk: () ⇒ Future[Chunk]): Future[Chunk] = { diff --git a/console-app/src/main/scala/com/karasiq/shadowcloud/console/SCConsoleConfig.scala b/console-app/src/main/scala/com/karasiq/shadowcloud/console/SCConsoleConfig.scala index 98a221ca..e821845a 100644 --- a/console-app/src/main/scala/com/karasiq/shadowcloud/console/SCConsoleConfig.scala +++ b/console-app/src/main/scala/com/karasiq/shadowcloud/console/SCConsoleConfig.scala @@ -10,19 +10,20 @@ object SCConsoleConfig { def load(): Config = { val autoParallelismConfig = { val cpuAvailable = Runtime.getRuntime.availableProcessors() - val parallelism = math.max(8, math.min(1, cpuAvailable / 2)) + val parallelism = math.max(8, math.min(1, cpuAvailable / 2)) ConfigFactory.parseString(s"shadowcloud.parallelism.default = $parallelism") } val substitutionsConfig = ConfigFactory.parseResourcesAnySyntax("sc-substitutions") - val defaultConfig = ConfigFactory.defaultOverrides() + val defaultConfig = ConfigFactory + .defaultOverrides() .withFallback(ConfigFactory.defaultApplication()) .withFallback(ConfigFactory.defaultReference()) val serverAppConfig = { val fileConfig = { - val configFilePath = sys.props.getOrElse("shadowcloud.external-config", "shadowcloud.conf") + val configFilePath = sys.props.getOrElse("shadowcloud.external-config", "shadowcloud.conf") val optionalConfFile = Paths.get(configFilePath) if (Files.isRegularFile(optionalConfFile)) ConfigFactory.parseFile(optionalConfFile.toFile) diff --git a/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/actors/RegionDispatcherTest.scala b/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/actors/RegionDispatcherTest.scala index 93c82c55..6a5dd39b 100644 --- a/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/actors/RegionDispatcherTest.scala +++ b/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/actors/RegionDispatcherTest.scala @@ -100,7 +100,7 @@ class RegionDispatcherTest extends SCExtensionSpec with FlatSpecLike with Sequen it should "create availability report" in { val report = (testRegion ? GetFileAvailability(folder.files.head.copy(chunks = Seq(chunk)))).mapTo[GetFileAvailability.Success].futureValue - report.result.chunksByStorage shouldBe Map(testStorageId → Set(chunk)) + report.result.chunksByStorage shouldBe Map(testStorageId → Set(chunk)) report.result.percentagesByStorage shouldBe Map(testStorageId → 100.0) } @@ -231,12 +231,15 @@ class RegionDispatcherTest extends SCExtensionSpec with FlatSpecLike with Sequen resultDiff.time should be > oldDiff.time } - override protected def beforeAll(): Unit = { super.beforeAll() storage ! StorageIndex.OpenIndex(testRegionId) testRegion ! RegionDispatcher.AttachStorage(testStorageId, storageProps, storage, initialHealth) - awaitAssert(RegionDispatcher.GetHealth.unwrapFuture(testRegion ? RegionDispatcher.GetHealth).futureValue shouldBe 'fullyOnline, 30 seconds, 1 second) + awaitAssert( + RegionDispatcher.GetHealth.unwrapFuture(testRegion ? RegionDispatcher.GetHealth).futureValue shouldBe 'fullyOnline, + 30 seconds, + 1 second + ) } private def storageUnsubscribe(): Unit = { diff --git a/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/actors/RegionGCTest.scala b/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/actors/RegionGCTest.scala index 33cbe2ca..fcb7df0f 100644 --- a/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/actors/RegionGCTest.scala +++ b/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/actors/RegionGCTest.scala @@ -13,7 +13,7 @@ import org.scalatest.{FlatSpecLike, SequentialNestedSuiteExecution} import scala.concurrent.duration._ class RegionGCTest extends SCExtensionSpec with FlatSpecLike with SequentialNestedSuiteExecution { - val testRegionId = "regionGCTest" + val testRegionId = "regionGCTest" val testStorageId = "regionGCTest" "Region GC" should "delete broken chunks" in { @@ -59,7 +59,6 @@ class RegionGCTest extends SCExtensionSpec with FlatSpecLike with SequentialNest } } - override protected def beforeAll(): Unit = { super.beforeAll() registerRegionAndStorages() diff --git a/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/actors/RegionRepairTest.scala b/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/actors/RegionRepairTest.scala index e7d9a5bc..2f782aed 100644 --- a/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/actors/RegionRepairTest.scala +++ b/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/actors/RegionRepairTest.scala @@ -23,14 +23,16 @@ class RegionRepairTest extends SCExtensionSpec with FlatSpecLike with Sequential sc.ops.region.writeChunk(testRegionId, chunk).futureValue shouldBe chunk sc.ops.region.synchronize(testRegionId).futureValue - val (repairStream, repairResult) = TestSource.probe[RegionRepairStream.Request] + val (repairStream, repairResult) = TestSource + .probe[RegionRepairStream.Request] .alsoTo(RegionRepairStream(sc.ops.region, sc.config.parallelism, sc.config.chunks.chunkSize)) .mapAsync(1)(_.result.future) .toMat(TestSink.probe)(Keep.both) .run() - repairStream.sendNext(RegionRepairStream.Request(testRegionId, - RegionRepairStream.Strategy.SetAffinity(ChunkWriteAffinity(Seq(testStorage1, testStorage2))))) + repairStream.sendNext( + RegionRepairStream.Request(testRegionId, RegionRepairStream.Strategy.SetAffinity(ChunkWriteAffinity(Seq(testStorage1, testStorage2)))) + ) val repaired = repairResult.requestNext() repaired shouldBe Seq(chunk) @@ -51,7 +53,8 @@ class RegionRepairTest extends SCExtensionSpec with FlatSpecLike with Sequential private[this] def registerRegionAndStorages(): Unit = { sc.ops.supervisor.createRegion(testRegionId, sc.configs.regionConfig(testRegionId)) - sc.ops.supervisor.createStorage(testStorage1, StorageProps.inMemory.copy(quota = Quota(ConfigProps.toConfig(ConfigProps("use-space-percent" → 33))))) + sc.ops.supervisor + .createStorage(testStorage1, StorageProps.inMemory.copy(quota = Quota(ConfigProps.toConfig(ConfigProps("use-space-percent" → 33))))) sc.ops.supervisor.createStorage(testStorage2, StorageProps.inMemory) sc.ops.supervisor.register(testRegionId, testStorage1) sc.ops.supervisor.register(testRegionId, testStorage2) diff --git a/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/actors/RegionSupervisorTest.scala b/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/actors/RegionSupervisorTest.scala index 0221e0fb..72cb04ad 100644 --- a/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/actors/RegionSupervisorTest.scala +++ b/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/actors/RegionSupervisorTest.scala @@ -13,9 +13,9 @@ import scala.concurrent.duration._ class RegionSupervisorTest extends SCExtensionSpec with FlatSpecLike with Matchers { import RegionSupervisor._ - val testRegion = "testRegion" + val testRegion = "testRegion" val testStorage = "testStorage" - val supervisor = system.actorOf(props, "supervisor") + val supervisor = system.actorOf(props, "supervisor") "Region supervisor" should "add region" in { supervisor ! CreateRegion(testRegion, CoreTestUtils.regionConfig("testRegion")) @@ -78,7 +78,7 @@ class RegionSupervisorTest extends SCExtensionSpec with FlatSpecLike with Matche it should "resume storage" in { supervisor ! ResumeStorage(testStorage) - val (regions, storages) = requestState() + val (regions, storages) = requestState() val StorageStatus(`testStorage`, _, ActorState.Active(_), storageRegions) = storages(testStorage) storageRegions shouldBe Set(testRegion) regions.keySet shouldBe Set(testRegion) @@ -95,7 +95,7 @@ class RegionSupervisorTest extends SCExtensionSpec with FlatSpecLike with Matche it should "resume region" in { supervisor ! ResumeRegion(testRegion) - val (regions, storages) = requestState() + val (regions, storages) = requestState() val RegionStatus(`testRegion`, _, ActorState.Active(_), regionStorages) = regions(testRegion) regionStorages shouldBe Set(testStorage) storages.keySet shouldBe Set(testStorage) @@ -116,4 +116,3 @@ class RegionSupervisorTest extends SCExtensionSpec with FlatSpecLike with Matche (regions, storages) } } - diff --git a/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/compression/LZ4StreamsTest.scala b/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/compression/LZ4StreamsTest.scala index 3d12cf1a..fe47574a 100644 --- a/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/compression/LZ4StreamsTest.scala +++ b/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/compression/LZ4StreamsTest.scala @@ -11,7 +11,8 @@ class LZ4StreamsTest extends ActorSpec with ActorSpecImplicits with FlatSpecLike val testBytes = TestUtils.indexedBytes._1 "LZ4" should "compress bytes" in { - val futureCompressed = Source.fromIterator(() ⇒ testBytes.grouped(100)) + val futureCompressed = Source + .fromIterator(() ⇒ testBytes.grouped(100)) .via(LZ4Streams.compress) .via(ByteStreams.concat) .runWith(Sink.head) @@ -19,7 +20,8 @@ class LZ4StreamsTest extends ActorSpec with ActorSpecImplicits with FlatSpecLike val compressed = futureCompressed.futureValue compressed should not be empty - val futureUncompressed = Source.fromIterator(() ⇒ compressed.grouped(33)) + val futureUncompressed = Source + .fromIterator(() ⇒ compressed.grouped(33)) .via(LZ4Streams.decompress) .via(ByteStreams.concat) .runWith(Sink.head) diff --git a/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/compression/StreamCompressionTest.scala b/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/compression/StreamCompressionTest.scala index b1a862fd..5a467260 100644 --- a/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/compression/StreamCompressionTest.scala +++ b/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/compression/StreamCompressionTest.scala @@ -19,7 +19,8 @@ class StreamCompressionTest extends ActorSpec with ActorSpecImplicits with FlatS val testBytes = TestUtils.indexedBytes._1 def testCompression(uncompressed: ByteString): ByteString = { - val futureCompressed = Source.fromIterator(() ⇒ uncompressed.grouped(100)) + val futureCompressed = Source + .fromIterator(() ⇒ uncompressed.grouped(100)) .via(StreamCompression.compress(compType)) .via(ByteStreams.concat) .runWith(Sink.head) @@ -30,7 +31,8 @@ class StreamCompressionTest extends ActorSpec with ActorSpecImplicits with FlatS } def testDecompression(compressed: ByteString, expected: ByteString): Unit = { - val futureUncompressed = Source.fromIterator(() ⇒ compressed.grouped(33)) + val futureUncompressed = Source + .fromIterator(() ⇒ compressed.grouped(33)) .via(StreamCompression.decompress) .via(ByteStreams.concat) .runWith(Sink.head) @@ -60,10 +62,10 @@ object StreamCompressionTest { def readTestVector(compType: CompressionType.Value): (ByteString, ByteString) = { ResourceUtils.getPathOption(s"compression-vectors/$compType") match { case Some(path) ⇒ - val inputStream = Files.newInputStream(path) + val inputStream = Files.newInputStream(path) val objectInputStream = new ObjectInputStream(inputStream) - val uncompressed = objectInputStream.readObject().asInstanceOf[ByteString] - val compressed = objectInputStream.readObject().asInstanceOf[ByteString] + val uncompressed = objectInputStream.readObject().asInstanceOf[ByteString] + val compressed = objectInputStream.readObject().asInstanceOf[ByteString] objectInputStream.close() (uncompressed, compressed) @@ -76,7 +78,7 @@ object StreamCompressionTest { val fileName = testVectorsFolder.resolve(compType.toString) if (!Files.exists(fileName)) { if (!Files.isDirectory(testVectorsFolder)) Files.createDirectories(testVectorsFolder) - val outputStream = Files.newOutputStream(fileName) + val outputStream = Files.newOutputStream(fileName) val objectOutputStream = new ObjectOutputStream(outputStream) objectOutputStream.writeObject(uncompressed) objectOutputStream.writeObject(compressed) diff --git a/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/crypto/EncryptionModuleTest.scala b/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/crypto/EncryptionModuleTest.scala index 27c592d7..fa7c6745 100644 --- a/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/crypto/EncryptionModuleTest.scala +++ b/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/crypto/EncryptionModuleTest.scala @@ -41,9 +41,9 @@ class EncryptionModuleTest extends FlatSpec with Matchers { } private[this] def testEncryption(module: EncryptionModule): Unit = { - val data = TestUtils.randomBytes(100) + val data = TestUtils.randomBytes(100) val parameters = module.createParameters() - val encrypted = module.encrypt(data, parameters) + val encrypted = module.encrypt(data, parameters) // encrypted should not be data encrypted.length should be >= data.length val decrypted = module.decrypt(encrypted, parameters) @@ -53,13 +53,13 @@ class EncryptionModuleTest extends FlatSpec with Matchers { private[this] def runCrossTest(method1: EncryptionMethod, method2: EncryptionMethod): Unit = { def toString(m: EncryptionMethod) = s"${m.provider.capitalize} (${m.algorithm})" - val module1 = modules.crypto.encryptionModule(method1) - val module2 = modules.crypto.encryptionModule(method2) + val module1 = modules.crypto.encryptionModule(method1) + val module2 = modules.crypto.encryptionModule(method2) s"${toString(method1)}" should s"create compatible data for ${toString(method2)}" in { - val data = TestUtils.randomBytes(100) + val data = TestUtils.randomBytes(100) val parameters = module1.createParameters() - val encrypted = module1.encrypt(data, parameters) - val decrypted = module2.decrypt(encrypted, parameters) + val encrypted = module1.encrypt(data, parameters) + val decrypted = module2.decrypt(encrypted, parameters) decrypted shouldBe data } } diff --git a/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/crypto/HashingModuleTest.scala b/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/crypto/HashingModuleTest.scala index 02e68601..57b11d7f 100644 --- a/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/crypto/HashingModuleTest.scala +++ b/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/crypto/HashingModuleTest.scala @@ -1,19 +1,19 @@ package com.karasiq.shadowcloud.test.crypto - - import akka.util.ByteString import com.karasiq.shadowcloud.model.crypto.HashingMethod import com.karasiq.shadowcloud.test.utils.{ByteStringImplicits, CoreTestUtils} import org.scalatest.{FlatSpec, Matchers} class HashingModuleTest extends FlatSpec with Matchers with ByteStringImplicits { - val testData = ByteString("The testing facilities described up to this point were aiming at formulating assertions about a system’s behavior. If a test fails, it is usually your job to find the cause, fix it and verify the test again. This process is supported by debuggers as well as logging, where the Akka toolkit offers the following options:") + val testData = ByteString( + "The testing facilities described up to this point were aiming at formulating assertions about a system’s behavior. If a test fails, it is usually your job to find the cause, fix it and verify the test again. This process is supported by debuggers as well as logging, where the Akka toolkit offers the following options:" + ) val hashes = Map( - "SHA1" → "b608b0697e31e807fff8a37ae0d0292e62a8bbc2", - "SHA256" → "299a5a1fef2c67b6af326841684e020405978a34f7924025c1f5a188d808f08f", - "SHA512" → "46d69344d4fab2e20441d49a31ba94fbdba3d551fbbeb4646a00d9fd0e4d1a4b27c5c7ff9fcc992f0d88675e4de0dce1a8c8420298a60789d7a76ddf841d8088", + "SHA1" → "b608b0697e31e807fff8a37ae0d0292e62a8bbc2", + "SHA256" → "299a5a1fef2c67b6af326841684e020405978a34f7924025c1f5a188d808f08f", + "SHA512" → "46d69344d4fab2e20441d49a31ba94fbdba3d551fbbeb4646a00d9fd0e4d1a4b27c5c7ff9fcc992f0d88675e4de0dce1a8c8420298a60789d7a76ddf841d8088", "Blake2b" → "f38fec1634ceaae44ca6a6ee6f493938b746a9d82581235ca6a748dc21ad865d" ) diff --git a/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/crypto/IndexEncryptionTest.scala b/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/crypto/IndexEncryptionTest.scala index bb19fa06..64957c2f 100644 --- a/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/crypto/IndexEncryptionTest.scala +++ b/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/crypto/IndexEncryptionTest.scala @@ -10,10 +10,12 @@ import com.karasiq.shadowcloud.test.utils.TestUtils class IndexEncryptionTest extends FlatSpec with Matchers { "Index encryption" should "create key hash" in { - IndexEncryption.getKeyHash(UUID.fromString("fb390d6a-545e-4ea7-a3ac-45e4a9223ebb"), - UUID.fromString("1ac2ad29-3edb-4f65-9142-cde158f06d28")) shouldBe (-1060735836) + IndexEncryption.getKeyHash( + UUID.fromString("fb390d6a-545e-4ea7-a3ac-45e4a9223ebb"), + UUID.fromString("1ac2ad29-3edb-4f65-9142-cde158f06d28") + ) shouldBe (-1060735836) } - + it should "create nonce" in { val nonce1 = TestUtils.indexedBytes._1.slice(0, 16) val nonce2 = TestUtils.indexedBytes._1.slice(16, 32) diff --git a/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/crypto/SignModuleTest.scala b/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/crypto/SignModuleTest.scala index e2dec92c..1dc300bd 100644 --- a/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/crypto/SignModuleTest.scala +++ b/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/crypto/SignModuleTest.scala @@ -1,7 +1,5 @@ package com.karasiq.shadowcloud.test.crypto - - import com.karasiq.shadowcloud.crypto.SignModule import com.karasiq.shadowcloud.model.crypto.{HashingMethod, SignMethod} import com.karasiq.shadowcloud.test.utils.CoreTestUtils.modules @@ -9,7 +7,7 @@ import com.karasiq.shadowcloud.test.utils.TestUtils import org.scalatest.{FlatSpec, Matchers} class SignModuleTest extends FlatSpec with Matchers { - val hashingMethod = HashingMethod("SHA-512") + val hashingMethod = HashingMethod("SHA-512") runTest("RSA", 1024) @@ -30,21 +28,21 @@ class SignModuleTest extends FlatSpec with Matchers { } private[this] def testSignature(module: SignModule): Unit = { - val data = TestUtils.randomBytes(100) + val data = TestUtils.randomBytes(100) val parameters = module.createParameters() - val signature = module.sign(data, parameters) + val signature = module.sign(data, parameters) module.verify(data, signature, parameters) shouldBe true } private[this] def runCrossTest(method1: SignMethod, method2: SignMethod): Unit = { def toString(m: SignMethod) = s"${m.provider.capitalize} (${m.algorithm})" - val module1 = modules.crypto.signModule(method1) - val module2 = modules.crypto.signModule(method2) + val module1 = modules.crypto.signModule(method1) + val module2 = modules.crypto.signModule(method2) s"${toString(method1)}" should s"create compatible signature for ${toString(method2)}" in { - val data = TestUtils.randomBytes(100) + val data = TestUtils.randomBytes(100) val parameters = module1.createParameters() - val signature = module1.sign(data, parameters) - val result = module2.verify(data, signature, parameters) + val signature = module1.sign(data, parameters) + val result = module2.verify(data, signature, parameters) result shouldBe true } } diff --git a/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/index/ChunkIndexTest.scala b/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/index/ChunkIndexTest.scala index 7469ccca..83efa283 100644 --- a/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/index/ChunkIndexTest.scala +++ b/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/index/ChunkIndexTest.scala @@ -1,7 +1,5 @@ package com.karasiq.shadowcloud.test.index - - import com.karasiq.shadowcloud.index.ChunkIndex import com.karasiq.shadowcloud.test.utils.CoreTestUtils import org.scalatest.{FlatSpec, Matchers} @@ -15,19 +13,19 @@ class ChunkIndexTest extends FlatSpec with Matchers { } it should "delete chunk" in { - val index = ChunkIndex(Set(chunk)) + val index = ChunkIndex(Set(chunk)) val index1 = index.deleteChunks(chunk) index1.chunks shouldBe empty } it should "delete chunk without data" in { - val index = ChunkIndex(Set(chunk)) + val index = ChunkIndex(Set(chunk)) val index1 = index.deleteChunks(chunk.withoutData) index1.chunks shouldBe empty } it should "merge" in { - val index = ChunkIndex(Set(chunk)) + val index = ChunkIndex(Set(chunk)) val chunk1 = CoreTestUtils.randomChunk val index1 = ChunkIndex(Set(chunk1)) val merged = index.merge(index1) diff --git a/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/index/FolderDiffTest.scala b/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/index/FolderDiffTest.scala index 0b8cf39d..6f2bb6d7 100644 --- a/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/index/FolderDiffTest.scala +++ b/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/index/FolderDiffTest.scala @@ -1,7 +1,5 @@ package com.karasiq.shadowcloud.test.index - - import com.karasiq.shadowcloud.index.FolderIndex import com.karasiq.shadowcloud.index.diffs.FolderIndexDiff import com.karasiq.shadowcloud.model.{Folder, Path} @@ -9,10 +7,10 @@ import com.karasiq.shadowcloud.test.utils.{CoreTestUtils, TestUtils} import org.scalatest.{FlatSpec, Matchers} class FolderDiffTest extends FlatSpec with Matchers { - val folder1 = CoreTestUtils.randomFolder() - val newFile = CoreTestUtils.randomFile(folder1.path) + val folder1 = CoreTestUtils.randomFolder() + val newFile = CoreTestUtils.randomFile(folder1.path) val newFolder = TestUtils.randomString - val folder2 = folder1.addFiles(newFile).addFolders(newFolder) + val folder2 = folder1.addFiles(newFile).addFolders(newFolder) "Folder diff" should "find new files" in { val diff = folder2.diff(folder1) @@ -25,7 +23,7 @@ class FolderDiffTest extends FlatSpec with Matchers { it should "find deleted files" in { val folder3 = CoreTestUtils.randomFolder(folder1.path) - val diff = folder3.diff(folder2) + val diff = folder3.diff(folder2) diff.newFiles shouldBe folder3.files diff.newFolders shouldBe folder3.folders diff.deletedFiles shouldBe folder2.files @@ -33,28 +31,30 @@ class FolderDiffTest extends FlatSpec with Matchers { } it should "modify folder" in { - val diff = folder2.diff(folder1) + val diff = folder2.diff(folder1) val folder3 = folder1.patch(diff) folder3 shouldBe folder2 } it should "modify folder index" in { - val index = FolderIndex(Seq(folder1)) - val diff = folder2.diff(folder1) + val index = FolderIndex(Seq(folder1)) + val diff = folder2.diff(folder1) val index1 = index.patch(FolderIndexDiff.fromDiffs(diff)) index1.folders shouldBe Map( - Path.root → Folder(Path.root, folder1.timestamp, Set(folder1.path.name)), + Path.root → Folder(Path.root, folder1.timestamp, Set(folder1.path.name)), folder1.path → folder2 - ).++(folder2.folders.map(name ⇒ (folder2.path / name) → Folder(folder2.path / name, folder2.timestamp)) ++ - folder1.folders.map(name ⇒ (folder1.path / name) → Folder(folder1.path / name, folder1.timestamp))) + ).++( + folder2.folders.map(name ⇒ (folder2.path / name) → Folder(folder2.path / name, folder2.timestamp)) ++ + folder1.folders.map(name ⇒ (folder1.path / name) → Folder(folder1.path / name, folder1.timestamp)) + ) } it should "add folder with parents" in { val index = FolderIndex.empty.patch(FolderIndexDiff.createFolders(Folder.create("/test1/test2/test3/test4"))) - index.folders("/").folders should contain ("test1") - index.folders("/test1").folders should contain ("test2") - index.folders("/test1/test2").folders should contain ("test3") - index.folders("/test1/test2/test3").folders should contain ("test4") + index.folders("/").folders should contain("test1") + index.folders("/test1").folders should contain("test2") + index.folders("/test1/test2").folders should contain("test3") + index.folders("/test1/test2/test3").folders should contain("test4") } it should "delete folder with children" in { @@ -69,7 +69,7 @@ class FolderDiffTest extends FlatSpec with Matchers { } it should "reverse" in { - val diff = folder2.diff(folder1) + val diff = folder2.diff(folder1) val folder3 = folder1.patch(diff) val reverse = diff.reverse reverse.time shouldBe diff.time @@ -83,9 +83,9 @@ class FolderDiffTest extends FlatSpec with Matchers { it should "merge" in { val folder3 = CoreTestUtils.randomFolder(folder1.path) - val diff = folder2.diff(folder1) // + Folder2 files - val diff1 = folder3.diff(folder2).copy(time = diff.time + 1) // - Folder1 files, - Folder2 files, + Folder3 files - val merged = diff.merge(diff1) // - Folder1 files, - Folder2 files, + Folder3 files + val diff = folder2.diff(folder1) // + Folder2 files + val diff1 = folder3.diff(folder2).copy(time = diff.time + 1) // - Folder1 files, - Folder2 files, + Folder3 files + val merged = diff.merge(diff1) // - Folder1 files, - Folder2 files, + Folder3 files merged.time shouldBe diff1.time merged.newFiles shouldBe folder3.files merged.newFolders shouldBe folder3.folders @@ -94,10 +94,10 @@ class FolderDiffTest extends FlatSpec with Matchers { } it should "diff" in { - val folder3 = CoreTestUtils.randomFolder(folder1.path) - val diff = folder2.diff(folder1) // + Folder2 files - val diff1 = folder3.diff(folder2) // - Folder1 files, - Folder2 files, + Folder3 files - val diffDiff = diff.diff(diff1) // + Folder1 files, + Folder2 files, - Folder3 files + val folder3 = CoreTestUtils.randomFolder(folder1.path) + val diff = folder2.diff(folder1) // + Folder2 files + val diff1 = folder3.diff(folder2) // - Folder1 files, - Folder2 files, + Folder3 files + val diffDiff = diff.diff(diff1) // + Folder1 files, + Folder2 files, - Folder3 files diffDiff.newFiles shouldBe (folder1.files ++ folder2.files -- folder3.files) diffDiff.newFolders shouldBe (folder1.folders ++ folder2.folders -- folder3.folders) diffDiff.deletedFiles shouldBe folder3.files diff --git a/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/index/FolderIndexTest.scala b/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/index/FolderIndexTest.scala index 9ea8fbd6..57098f42 100644 --- a/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/index/FolderIndexTest.scala +++ b/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/index/FolderIndexTest.scala @@ -13,7 +13,7 @@ class FolderIndexTest extends FlatSpec with Matchers { val folder4 = Folder.create("/f1/f2/f33") val folder5 = Folder.create("/f1/f2/f3/f4") val folder6 = Folder.create("/f2/f1") - val index = FolderIndex(Seq(folder1, folder2, folder3, folder4, folder5, folder6)) + val index = FolderIndex(Seq(folder1, folder2, folder3, folder4, folder5, folder6)) FolderIndex.traverseFolderTree(index, "/f1").toList shouldBe Seq(folder1, folder2, folder3, folder5, folder4) } } diff --git a/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/index/FolderTest.scala b/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/index/FolderTest.scala index 1b5a4077..d1a59b29 100644 --- a/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/index/FolderTest.scala +++ b/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/index/FolderTest.scala @@ -1,7 +1,5 @@ package com.karasiq.shadowcloud.test.index - - import com.karasiq.shadowcloud.model.{Folder, Timestamp} import com.karasiq.shadowcloud.test.utils.{CoreTestUtils, TestUtils} import org.scalatest.{FlatSpec, Matchers} @@ -11,7 +9,7 @@ class FolderTest extends FlatSpec with Matchers { "Folder" should "add file" in { val testFile = CoreTestUtils.randomFile(folder.path) - val folder1 = folder.addFiles(testFile) + val folder1 = folder.addFiles(testFile) folder1.timestamp.created shouldBe folder.timestamp.created folder1.timestamp.lastModified should be > folder.timestamp.lastModified folder1.files shouldBe Set(testFile) @@ -24,22 +22,22 @@ class FolderTest extends FlatSpec with Matchers { it should "remove file" in { val testFile = CoreTestUtils.randomFile(folder.path) - val folder1 = folder.addFiles(testFile) - val folder2 = folder.deleteFiles(folder1.files.head) + val folder1 = folder.addFiles(testFile) + val folder2 = folder.deleteFiles(folder1.files.head) folder2.timestamp.lastModified should be > folder.timestamp.lastModified folder2.timestamp.lastModified should be >= folder1.timestamp.lastModified folder2.files shouldBe empty } it should "add subdirectory" in { - val subDir = TestUtils.randomString + val subDir = TestUtils.randomString val folder1 = folder.addFolders(subDir) folder1.timestamp.lastModified should be > folder.timestamp.lastModified folder1.folders shouldBe Set(subDir) } it should "remove subdirectory" in { - val subDir = TestUtils.randomString + val subDir = TestUtils.randomString val folder1 = folder.addFolders(subDir) val folder2 = folder1.deleteFolders(subDir) folder2.timestamp.lastModified should be > folder.timestamp.lastModified diff --git a/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/index/HashCodesTest.scala b/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/index/HashCodesTest.scala index 57962244..35eb79f8 100644 --- a/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/index/HashCodesTest.scala +++ b/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/index/HashCodesTest.scala @@ -1,7 +1,5 @@ package com.karasiq.shadowcloud.test.index - - import akka.util.ByteString import com.karasiq.shadowcloud.storage.utils.IndexMerger.RegionKey import com.karasiq.shadowcloud.test.utils.{CoreTestUtils, TestUtils} @@ -19,14 +17,14 @@ class HashCodesTest extends FlatSpec with Matchers { val chunk6 = chunk2.copy(checksum = chunk2.checksum.copy(size = chunk2.checksum.size + 1)) chunk1 shouldBe chunk2 chunk1 shouldBe chunk3 - chunk1 shouldNot be (chunk4) - chunk1 shouldNot be (chunk5) - chunk1 shouldNot be (chunk6) + chunk1 shouldNot be(chunk4) + chunk1 shouldNot be(chunk5) + chunk1 shouldNot be(chunk6) chunk1.hashCode() shouldBe chunk2.hashCode() chunk1.hashCode() shouldBe chunk3.hashCode() chunk1.hashCode() shouldBe chunk4.hashCode() - chunk1.hashCode() shouldNot be (chunk5.hashCode()) - chunk1.hashCode() shouldNot be (chunk6.hashCode()) + chunk1.hashCode() shouldNot be(chunk5.hashCode()) + chunk1.hashCode() shouldNot be(chunk6.hashCode()) } "Files" should "be compared" in { @@ -37,9 +35,9 @@ class HashCodesTest extends FlatSpec with Matchers { val file5 = file1.copy(checksum = file1.checksum.copy(size = file1.checksum.size + 1)) val file6 = file1.copy(chunks = file1.chunks :+ CoreTestUtils.randomChunk) file1 shouldBe file2 - file1 shouldNot be (file3) - file1 shouldNot be (file4) - file1 shouldNot be (file5) + file1 shouldNot be(file3) + file1 shouldNot be(file4) + file1 shouldNot be(file5) file1.hashCode() shouldBe file2.hashCode() file1.hashCode() shouldBe file3.hashCode() // file1.hashCode() shouldNot be (file5.hashCode()) @@ -53,11 +51,11 @@ class HashCodesTest extends FlatSpec with Matchers { val folder4 = folder1.copy(folders = folder1.folders + TestUtils.randomString) val folder5 = folder1.copy(files = folder1.files + CoreTestUtils.randomFile(folder1.path)) folder2 shouldBe folder1 - folder3 shouldNot be (folder1) - folder4 shouldNot be (folder1) - folder5 shouldNot be (folder1) + folder3 shouldNot be(folder1) + folder4 shouldNot be(folder1) + folder5 shouldNot be(folder1) folder2.hashCode() shouldBe folder1.hashCode() - folder3.hashCode() shouldNot be (folder1.hashCode()) + folder3.hashCode() shouldNot be(folder1.hashCode()) // folder4.hashCode() shouldNot be (folder1.hashCode()) // folder5.hashCode() shouldNot be (folder1.hashCode()) } diff --git a/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/index/IndexMergerTest.scala b/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/index/IndexMergerTest.scala index a86c9e87..1dffe6d3 100644 --- a/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/index/IndexMergerTest.scala +++ b/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/index/IndexMergerTest.scala @@ -1,7 +1,5 @@ package com.karasiq.shadowcloud.test.index - - import com.karasiq.shadowcloud.exceptions.SCExceptions import com.karasiq.shadowcloud.index.FolderIndex import com.karasiq.shadowcloud.index.diffs.{FolderIndexDiff, IndexDiff} @@ -14,7 +12,7 @@ class IndexMergerTest extends WordSpec with Matchers { "Index" when { "empty" should { val index = IndexMerger.sequential() - val diff = TestUtils.testDiff + val diff = TestUtils.testDiff "add pending diff" in { index.addPending(diff) @@ -42,9 +40,9 @@ class IndexMergerTest extends WordSpec with Matchers { } "not empty" should { - val index = IndexMerger.sequential() - val diff1 = TestUtils.testDiff - val diff2 = CoreTestUtils.randomDiff + val index = IndexMerger.sequential() + val diff1 = TestUtils.testDiff + val diff2 = CoreTestUtils.randomDiff val diff2Reverse = diff2.reverse.copy(time = diff2.time + 1) "add initial diff" in { @@ -66,12 +64,14 @@ class IndexMergerTest extends WordSpec with Matchers { "reverse diff" in { index.add(diff2Reverse.time, diff2Reverse) index.diffs shouldBe Map(diff1.time → diff1, diff2.time → diff2, diff2Reverse.time → diff2Reverse) - index.mergedDiff.creates shouldBe diff1.copy(time = diff2Reverse.time, - folders = diff1.folders.copy(diff1.folders.folders.map(f ⇒ f.copy(time = diff2.time)))) + index.mergedDiff.creates shouldBe diff1.copy( + time = diff2Reverse.time, + folders = diff1.folders.copy(diff1.folders.folders.map(f ⇒ f.copy(time = diff2.time))) + ) index.chunks.chunks shouldBe diff1.chunks.newChunks index.folders shouldBe FolderIndex.empty.patch(diff1.folders).patch(diff2.folders).patch(diff2Reverse.folders) } - + "throw exception on diff rewrite" in { intercept[SCExceptions.DiffConflict] { index.add(diff1.time, diff1.merge(CoreTestUtils.randomDiff)) @@ -103,8 +103,13 @@ class IndexMergerTest extends WordSpec with Matchers { "delete folder tree" in { val index = IndexMerger.sequential() - val diff1 = IndexDiff(folders = FolderIndexDiff.createFolders(Folder("/1", files = Set(CoreTestUtils.randomFile("/1"))), - Folder("/1/2", files = Set(CoreTestUtils.randomFile("/1/2"))), Folder("/1/2/3", files = Set(CoreTestUtils.randomFile("/1/2/3"))))) + val diff1 = IndexDiff(folders = + FolderIndexDiff.createFolders( + Folder("/1", files = Set(CoreTestUtils.randomFile("/1"))), + Folder("/1/2", files = Set(CoreTestUtils.randomFile("/1/2"))), + Folder("/1/2/3", files = Set(CoreTestUtils.randomFile("/1/2/3"))) + ) + ) val diff2 = IndexDiff(folders = FolderIndexDiff.deleteFolderPaths("/1")) index.addPending(diff1) @@ -118,8 +123,12 @@ class IndexMergerTest extends WordSpec with Matchers { "delete nested file" in { val index = IndexMerger.sequential() - val diff1 = IndexDiff(folders = FolderIndexDiff.createFolders(Folder("/1", files = Set(CoreTestUtils.randomFile("/1"))), - Folder("/1/2/4/5/6/7/8", files = Set(CoreTestUtils.randomFile("/1/2/4/5/6/7/8"))))) + val diff1 = IndexDiff(folders = + FolderIndexDiff.createFolders( + Folder("/1", files = Set(CoreTestUtils.randomFile("/1"))), + Folder("/1/2/4/5/6/7/8", files = Set(CoreTestUtils.randomFile("/1/2/4/5/6/7/8"))) + ) + ) val diff2 = IndexDiff(folders = FolderIndexDiff.deleteFolderPaths("/1")) index.addPending(diff1) diff --git a/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/providers/SessionProviderTest.scala b/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/providers/SessionProviderTest.scala index 5c3157ed..3b795dad 100644 --- a/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/providers/SessionProviderTest.scala +++ b/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/providers/SessionProviderTest.scala @@ -8,8 +8,8 @@ import com.karasiq.shadowcloud.test.utils.{SCExtensionSpec, TestUtils} class SessionProviderTest extends SCExtensionSpec with FlatSpecLike { val storageId = "testStorage" - val testKey = "test_session" - val testData = TestUtils.randomBytes(10) + val testKey = "test_session" + val testData = TestUtils.randomBytes(10) val testValue = TestUtils.testDiff "Session provider" should "create session" in { diff --git a/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/serialization/SerializationModuleTest.scala b/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/serialization/SerializationModuleTest.scala index 2d786afd..12cc74eb 100644 --- a/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/serialization/SerializationModuleTest.scala +++ b/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/serialization/SerializationModuleTest.scala @@ -18,19 +18,19 @@ class SerializationModuleTest extends SCExtensionSpec with FlatSpecLike { private[this] def testModule(name: String, module: SerializationModule): Unit = { s"${name.capitalize} serializer" should "serialize config" in { val config = CoreTestUtils.config.rootConfig - val bytes = module.toBytes(config) + val bytes = module.toBytes(config) module.fromBytes[Config](bytes) shouldBe config } it should "serialize wrapped config" in { val config = CoreTestUtils.config - val bytes = module.toBytes(config) + val bytes = module.toBytes(config) module.fromBytes[SCConfig](bytes) shouldBe config } it should "serialize protobuf message" in { val message = SerializedIndexData(data = TestUtils.randomBytes(50)) - val bytes = module.toBytes(message) + val bytes = module.toBytes(message) module.fromBytes[SerializedIndexData](bytes) shouldBe message } @@ -41,7 +41,7 @@ class SerializationModuleTest extends SCExtensionSpec with FlatSpecLike { } it should "serialize file" in { - val file = CoreTestUtils.randomFile() + val file = CoreTestUtils.randomFile() val bytes = module.toBytes(file) val file1 = module.fromBytes[File](bytes) file1 shouldBe file @@ -50,20 +50,20 @@ class SerializationModuleTest extends SCExtensionSpec with FlatSpecLike { it should "serialize folder" in { val folder = CoreTestUtils.randomFolder() - val bytes = module.toBytes(folder) + val bytes = module.toBytes(folder) module.fromBytes[Folder](bytes) shouldBe folder } it should "serialize diff" in { - val diff = TestUtils.testDiff + val diff = TestUtils.testDiff val bytes = module.toBytes(diff) module.fromBytes[IndexDiff](bytes) shouldBe diff println(bytes.toHexString) } it should "read test diff" in { - val diff = TestUtils.testDiff - val bytes = Source.fromResource(s"test-diff-$name.txt").mkString.trim() + val diff = TestUtils.testDiff + val bytes = Source.fromResource(s"test-diff-$name.txt").mkString.trim() val deserializedDiff = module.fromBytes[IndexDiff](ByteString.fromHexString(bytes)) deserializedDiff shouldBe diff } diff --git a/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/storage/ChunkKeyMapperTest.scala b/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/storage/ChunkKeyMapperTest.scala index 730ea864..039e8070 100644 --- a/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/storage/ChunkKeyMapperTest.scala +++ b/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/storage/ChunkKeyMapperTest.scala @@ -42,11 +42,13 @@ class ChunkKeyMapperTest extends FlatSpec with Matchers { } it should "create multi-iteration HMAC" in { - val config = ConfigProps.toConfig(ConfigProps( - "class" → "com.karasiq.shadowcloud.storage.utils.mappers.HashNonceHMACKeyMapper", - "algorithm" → "HmacSHA1", - "iterations" → 100 - )) + val config = ConfigProps.toConfig( + ConfigProps( + "class" → "com.karasiq.shadowcloud.storage.utils.mappers.HashNonceHMACKeyMapper", + "algorithm" → "HmacSHA1", + "iterations" → 100 + ) + ) val mapper = ChunkKeyMapper.forConfig(config) mapper(testChunk.copy(encryption = TestUtils.testSymmetricParameters)) shouldBe @@ -54,20 +56,28 @@ class ChunkKeyMapperTest extends FlatSpec with Matchers { } "Composite key mapper" should "create composite key" in { - val mapper = new CompositeKeyMapper(ConfigProps.toConfig(ConfigProps( - "mappers" → Seq("hmac-mapper", "hash").asJava, - "hmac-mapper" → Map("class" → "com.karasiq.shadowcloud.storage.utils.mappers.HashNonceHMACKeyMapper", "algorithm" → "HmacSHA1").asJava - ))) + val mapper = new CompositeKeyMapper( + ConfigProps.toConfig( + ConfigProps( + "mappers" → Seq("hmac-mapper", "hash").asJava, + "hmac-mapper" → Map("class" → "com.karasiq.shadowcloud.storage.utils.mappers.HashNonceHMACKeyMapper", "algorithm" → "HmacSHA1").asJava + ) + ) + ) mapper(testChunk.copy(encryption = TestUtils.testSymmetricParameters)) shouldBe HexString.decode("d335659cc935ba592443781cfeda75c8181574d4f660847d03634f41c45f7be337b02973a083721a") } it should "create composite XOR key" in { - val mapper = new CompositeKeyMapper(ConfigProps.toConfig(ConfigProps( - "strategy" → "xor", - "mappers" → Seq("hmac-mapper", "hash").asJava, - "hmac-mapper" → Map("class" → "com.karasiq.shadowcloud.storage.utils.mappers.HashNonceHMACKeyMapper", "algorithm" → "HmacSHA1").asJava - ))) + val mapper = new CompositeKeyMapper( + ConfigProps.toConfig( + ConfigProps( + "strategy" → "xor", + "mappers" → Seq("hmac-mapper", "hash").asJava, + "hmac-mapper" → Map("class" → "com.karasiq.shadowcloud.storage.utils.mappers.HashNonceHMACKeyMapper", "algorithm" → "HmacSHA1").asJava + ) + ) + ) mapper(testChunk.copy(encryption = TestUtils.testSymmetricParameters)) shouldBe HexString.decode("2555e1e1ca56f518e01c03ffc96a5cbbb89606ce") } diff --git a/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/storage/RepositoryTest.scala b/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/storage/RepositoryTest.scala index 355cd799..126d4ea1 100644 --- a/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/storage/RepositoryTest.scala +++ b/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/storage/RepositoryTest.scala @@ -26,11 +26,12 @@ class RepositoryTest extends SCExtensionSpec with FlatSpecLike { } */ private[this] def testRepository(repository: KeyValueRepository): Unit = { - val chunk = CoreTestUtils.randomChunk + val chunk = CoreTestUtils.randomChunk val testRepository = RepositoryKeys.toHexString(repository) // Write chunk - val (write, writeResult) = TestSource.probe[ByteString] + val (write, writeResult) = TestSource + .probe[ByteString] .toMat(testRepository.write(chunk.checksum.hash))(Keep.both) .run() write.sendNext(chunk.data.plain) @@ -47,7 +48,8 @@ class RepositoryTest extends SCExtensionSpec with FlatSpecLike { keys.expectComplete() // Read chunk - val read = testRepository.read(chunk.checksum.hash) + val read = testRepository + .read(chunk.checksum.hash) .via(ByteStreams.concat) .runWith(TestSink.probe) @@ -56,7 +58,8 @@ class RepositoryTest extends SCExtensionSpec with FlatSpecLike { // Rewrite error val rewriteBytes = TestUtils.randomBytes(chunk.data.plain.length) - val rewriteResult = Source.single(rewriteBytes) + val rewriteResult = Source + .single(rewriteBytes) .runWith(testRepository.write(chunk.checksum.hash)) whenReady(rewriteResult) { result ⇒ diff --git a/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/streams/AkkaStreamUtilsTest.scala b/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/streams/AkkaStreamUtilsTest.scala index 2a0e86ab..a4d3ecf5 100644 --- a/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/streams/AkkaStreamUtilsTest.scala +++ b/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/streams/AkkaStreamUtilsTest.scala @@ -9,11 +9,10 @@ import org.scalatest.FlatSpecLike import com.karasiq.shadowcloud.streams.utils.AkkaStreamUtils import com.karasiq.shadowcloud.test.utils.{ActorSpec, ActorSpecImplicits} - class AkkaStreamUtilsTest extends ActorSpec with ActorSpecImplicits with FlatSpecLike { "Input stream" should "be created" in { val probe = Source(List(ByteString(1, 2, 3, 4, 5), ByteString(6, 7, 8, 9, 10))) - .via(AkkaStreamUtils.writeInputStream { inputStream ⇒ + .via(AkkaStreamUtils.writeInputStream { inputStream ⇒ val bytes = ByteString.fromArrayUnsafe(IOUtils.toByteArray(inputStream)) Source.single(bytes) }) diff --git a/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/streams/ChunkRangesTest.scala b/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/streams/ChunkRangesTest.scala index c03019ea..5d128502 100644 --- a/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/streams/ChunkRangesTest.scala +++ b/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/streams/ChunkRangesTest.scala @@ -56,7 +56,7 @@ class ChunkRangesTest extends FlatSpec with Matchers { it should "split byte string" in { val range1 = ChunkRanges.Range(10, 20) - val bytes = TestUtils.randomBytes(20) + val bytes = TestUtils.randomBytes(20) range1.slice(bytes) shouldBe bytes.drop(10).take(10) } diff --git a/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/streams/ChunkSplitterTest.scala b/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/streams/ChunkSplitterTest.scala index 9f9dbfb1..65cd9f38 100644 --- a/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/streams/ChunkSplitterTest.scala +++ b/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/streams/ChunkSplitterTest.scala @@ -1,7 +1,5 @@ package com.karasiq.shadowcloud.test.streams - - import akka.stream.scaladsl.{Keep, Sink, Source} import akka.stream.testkit.scaladsl.{TestSink, TestSource} import akka.util.ByteString @@ -13,12 +11,13 @@ import org.scalatest.FlatSpecLike //noinspection ZeroIndexToHead class ChunkSplitterTest extends SCExtensionSpec with FlatSpecLike { val (sourceBytes, sourceFile) = TestUtils.indexedBytes - val hashingMethod = sourceFile.checksum.method - val sourceHashes = sourceFile.chunks.map(_.checksum.hash) - val chunkProcessing = sc.streams.chunk + val hashingMethod = sourceFile.checksum.method + val sourceHashes = sourceFile.chunks.map(_.checksum.hash) + val chunkProcessing = sc.streams.chunk "Chunk splitter" should "split text" in { - val fullOut = Source.single(sourceBytes) + val fullOut = Source + .single(sourceBytes) .via(ChunkSplitter(100)) .via(chunkProcessing.createHashes(hashingMethod)) .map(_.checksum.hash) @@ -28,7 +27,8 @@ class ChunkSplitterTest extends SCExtensionSpec with FlatSpecLike { } it should "join text" in { - val (in, out) = TestSource.probe[ByteString] + val (in, out) = TestSource + .probe[ByteString] .via(ChunkSplitter(100)) .via(chunkProcessing.createHashes(hashingMethod, hashingMethod)) .map(_.checksum.hash) @@ -48,18 +48,23 @@ class ChunkSplitterTest extends SCExtensionSpec with FlatSpecLike { "Chunk encryptor" should "encrypt chunk stream" in { def testChunk(chunk: Chunk) = { - val hasher = sc.modules.crypto.hashingModule(chunk.checksum.method) + val hasher = sc.modules.crypto.hashingModule(chunk.checksum.method) val decryptor = sc.modules.crypto.encryptionModule(chunk.encryption.method) - val hash1 = hasher.createHash(chunk.data.plain) - val hash2 = hasher.createHash(chunk.data.encrypted) - val size1 = chunk.data.plain.length - val size2 = chunk.data.encrypted.length - val data = decryptor.decrypt(chunk.data.encrypted, chunk.encryption) - chunk shouldBe Chunk(Checksum(chunk.checksum.method, chunk.checksum.method, size1, hash1, size2, hash2), chunk.encryption, Data(data, chunk.data.encrypted)) + val hash1 = hasher.createHash(chunk.data.plain) + val hash2 = hasher.createHash(chunk.data.encrypted) + val size1 = chunk.data.plain.length + val size2 = chunk.data.encrypted.length + val data = decryptor.decrypt(chunk.data.encrypted, chunk.encryption) + chunk shouldBe Chunk( + Checksum(chunk.checksum.method, chunk.checksum.method, size1, hash1, size2, hash2), + chunk.encryption, + Data(data, chunk.data.encrypted) + ) chunk } - val result = Source.single(sourceBytes) + val result = Source + .single(sourceBytes) .via(ChunkSplitter(100)) .via(chunkProcessing.beforeWrite(hashing = hashingMethod, encHashing = hashingMethod)) .map(testChunk) diff --git a/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/streams/IndexRepositoryStreamsTest.scala b/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/streams/IndexRepositoryStreamsTest.scala index 98a5fcd2..9f5e4971 100644 --- a/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/streams/IndexRepositoryStreamsTest.scala +++ b/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/streams/IndexRepositoryStreamsTest.scala @@ -37,12 +37,13 @@ class IndexRepositoryStreamsTest extends SCExtensionSpec with FlatSpecLike { } private[this] def testRepository(repository: KeyValueRepository): Unit = { - val diff = CoreTestUtils.randomDiff + val diff = CoreTestUtils.randomDiff val testRepository = RepositoryKeys.toLong(repository) // Write diff val streams = IndexRepositoryStreams("testRegion", CoreTestUtils.storageConfig("testStorage"), system) - val (write, writeResult) = TestSource.probe[(Long, IndexData)] + val (write, writeResult) = TestSource + .probe[(Long, IndexData)] .via(streams.write(testRepository)) .toMat(TestSink.probe)(Keep.both) .run() @@ -59,7 +60,8 @@ class IndexRepositoryStreamsTest extends SCExtensionSpec with FlatSpecLike { keys.expectComplete() // Read diff - val (read, readResult) = TestSource.probe[Long] + val (read, readResult) = TestSource + .probe[Long] .via(streams.read(testRepository)) .toMat(TestSink.probe)(Keep.both) .run() @@ -70,19 +72,21 @@ class IndexRepositoryStreamsTest extends SCExtensionSpec with FlatSpecLike { readResult.expectComplete() // Rewrite error - val rewriteResult = Source.single(TestUtils.randomBytes(200)) + val rewriteResult = Source + .single(TestUtils.randomBytes(200)) .runWith(testRepository.write(diff.time)) whenReady(rewriteResult) { result ⇒ result.isFailure shouldBe true } - val deleteResult = Source.single(diff.time) + val deleteResult = Source + .single(diff.time) .via(streams.delete(testRepository)) .runWith(Sink.head) whenReady(deleteResult) { result ⇒ - val IndexIOResult(key, _, StorageIOResult.Success(_, count)) = result + val IndexIOResult(key, _, StorageIOResult.Success(_, count)) = result key shouldBe diff.time count shouldBe diffBytes val keys = testRepository.keys.runWith(TestSink.probe) diff --git a/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/streams/MetadataStreamsTest.scala b/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/streams/MetadataStreamsTest.scala index 11afa9ce..46b525f9 100644 --- a/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/streams/MetadataStreamsTest.scala +++ b/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/streams/MetadataStreamsTest.scala @@ -20,10 +20,11 @@ object MetadataStreamsTest { class MetadataStreamsTest extends SCExtensionSpec with FlatSpecLike with SequentialNestedSuiteExecution { "Mime detector" should "detect mime type" in { - val testOut = MetadataStreamsTest.testJpegStream() + val testOut = MetadataStreamsTest + .testJpegStream() .via(MimeDetectorStream(sc.modules.metadata, "14935431092820.jpg", 10000)) .runWith(TestSink.probe) - + val mimeType = testOut.requestNext() mimeType shouldBe "image/jpeg" @@ -31,14 +32,17 @@ class MetadataStreamsTest extends SCExtensionSpec with FlatSpecLike with Sequent testOut.expectComplete() } - val testRegionId = "metadataStreamsTest" + val testRegionId = "metadataStreamsTest" val testStorageId = "metadataStreamsTest" - val testFileId = FileId.create() - val testMetadata = Metadata(Some(Metadata.Tag("test", "test", Metadata.Tag.Disposition.PREVIEW)), - Metadata.Value.Thumbnail(Metadata.Thumbnail("random", TestUtils.randomBytes(100)))) + val testFileId = FileId.create() + val testMetadata = Metadata( + Some(Metadata.Tag("test", "test", Metadata.Tag.Disposition.PREVIEW)), + Metadata.Value.Thumbnail(Metadata.Thumbnail("random", TestUtils.randomBytes(100))) + ) "Metadata streams" should "write metadata" in { - val (testIn, testOut) = TestSource.probe[Metadata] + val (testIn, testOut) = TestSource + .probe[Metadata] .via(sc.streams.metadata.writeAll(testRegionId, testFileId)) .toMat(TestSink.probe)(Keep.both) .run() @@ -55,7 +59,8 @@ class MetadataStreamsTest extends SCExtensionSpec with FlatSpecLike with Sequent } it should "read metadata" in { - val testOut = sc.streams.metadata.read(testRegionId, testFileId, Metadata.Tag.Disposition.PREVIEW) + val testOut = sc.streams.metadata + .read(testRegionId, testFileId, Metadata.Tag.Disposition.PREVIEW) .runWith(TestSink.probe) testOut.request(2) @@ -66,7 +71,8 @@ class MetadataStreamsTest extends SCExtensionSpec with FlatSpecLike with Sequent it should "delete metadata" in { sc.streams.metadata.delete(testRegionId, testFileId).futureValue.files should not be empty - val testOut = sc.streams.metadata.read(testRegionId, testFileId, Metadata.Tag.Disposition.PREVIEW) + val testOut = sc.streams.metadata + .read(testRegionId, testFileId, Metadata.Tag.Disposition.PREVIEW) .runWith(TestSink.probe) testOut.request(1) @@ -74,7 +80,8 @@ class MetadataStreamsTest extends SCExtensionSpec with FlatSpecLike with Sequent } it should "create metadata" in { - val testTags = MetadataStreamsTest.testJpegStream() + val testTags = MetadataStreamsTest + .testJpegStream() .via(sc.streams.metadata.create("test.jpg")) .runWith(Sink.seq) .futureValue(Timeout(10 seconds)) @@ -89,7 +96,8 @@ class MetadataStreamsTest extends SCExtensionSpec with FlatSpecLike with Sequent } it should "write metadata on the fly" in { - val testFileStream = MetadataStreamsTest.testJpegStream() + val testFileStream = MetadataStreamsTest + .testJpegStream() .via(sc.streams.metadata.writeFileAndMetadata(testRegionId, Path.root / "test.jpg")) .runWith(TestSink.probe) diff --git a/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/streams/RangedChunkStreamTest.scala b/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/streams/RangedChunkStreamTest.scala index 468876ca..9eefbe54 100644 --- a/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/streams/RangedChunkStreamTest.scala +++ b/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/streams/RangedChunkStreamTest.scala @@ -11,7 +11,7 @@ import org.scalatest.{FlatSpecLike, SequentialNestedSuiteExecution} import scala.concurrent.duration._ class RangedChunkStreamTest extends SCExtensionSpec with FlatSpecLike with SequentialNestedSuiteExecution { - val testRegion = "testRegion" + val testRegion = "testRegion" val (testBytes, testFile) = TestUtils.indexedBytes "Ranged chunk stream" should "be read" in { diff --git a/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/utils/ByteStringUnsafeTest.scala b/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/utils/ByteStringUnsafeTest.scala index 27d2401d..aab3c719 100644 --- a/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/utils/ByteStringUnsafeTest.scala +++ b/core/assembly/src/test/scala/com/karasiq/shadowcloud/test/utils/ByteStringUnsafeTest.scala @@ -7,8 +7,8 @@ import com.karasiq.shadowcloud.utils.ByteStringUnsafe class ByteStringUnsafeTest extends FlatSpec { "ByteString array" should "be extracted" in { - val array = new Array[Byte](4) - val bs = ByteString.fromArrayUnsafe(array) + val array = new Array[Byte](4) + val bs = ByteString.fromArrayUnsafe(array) val extractedArray = ByteStringUnsafe.getArray(bs) assert(extractedArray eq array, "Arrays not equal") } diff --git a/core/src/main/scala/com/karasiq/shadowcloud/actors/ChunkIODispatcher.scala b/core/src/main/scala/com/karasiq/shadowcloud/actors/ChunkIODispatcher.scala index 30928d37..7a52d5e0 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/actors/ChunkIODispatcher.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/actors/ChunkIODispatcher.scala @@ -83,38 +83,40 @@ private final class ChunkIODispatcher(storageId: StorageId, storageProps: Storag private[this] val writeQueue = Source .queue[(ChunkPath, Chunk, Promise[StorageIOResult])](config.chunkIO.writeQueueSize, OverflowStrategy.dropNew) .flatMapMerge( - config.chunkIO.writeParallelism, { - case (path, chunk, promise) ⇒ - val repository = subRepository(path.regionId) - val writeSource = Source - .single(chunk.data.encrypted) - .alsoToMat(repository.write(path.chunkId))(Keep.right) //.alsoToMat(RepositoryStreams.writeWithRetries(repository, path.chunkId))(Keep.right) - - writeSource.extractMatFuture - .flatMapConcat { - case res @ StorageIOResult.Failure(_, StorageException.AlreadyExists(sp, _)) ⇒ - log.warning("Chunk already exists: {}", sp) - - if (config.immutable) - Source.single(res) - else - Source - .single(path.chunkId) - .mapAsync(1)(repository.delete(_)) - .flatMapConcatMatRight(_ ⇒ writeSource) - .extractMatFuture - .map(_.last) - - case otherResult ⇒ - Source.single(otherResult) - } - .completionTimeout(config.chunkIO.writeTimeout) - .alsoTo(AkkaStreamUtils.successPromiseOnFirst(promise)) - .viaMat(AkkaStreamUtils.ignoreUpstream(Source.future(promise.future)))(Keep.right) - .log("storage-chunkio-write") - .map(_ ⇒ NotUsed) - .recoverWithRetries(1, { case _ ⇒ Source.empty }) - .named("storageWrite") + config.chunkIO.writeParallelism, + { case (path, chunk, promise) ⇒ + val repository = subRepository(path.regionId) + val writeSource = Source + .single(chunk.data.encrypted) + .alsoToMat(repository.write(path.chunkId))( + Keep.right + ) //.alsoToMat(RepositoryStreams.writeWithRetries(repository, path.chunkId))(Keep.right) + + writeSource.extractMatFuture + .flatMapConcat { + case res @ StorageIOResult.Failure(_, StorageException.AlreadyExists(sp, _)) ⇒ + log.warning("Chunk already exists: {}", sp) + + if (config.immutable) + Source.single(res) + else + Source + .single(path.chunkId) + .mapAsync(1)(repository.delete(_)) + .flatMapConcatMatRight(_ ⇒ writeSource) + .extractMatFuture + .map(_.last) + + case otherResult ⇒ + Source.single(otherResult) + } + .completionTimeout(config.chunkIO.writeTimeout) + .alsoTo(AkkaStreamUtils.successPromiseOnFirst(promise)) + .viaMat(AkkaStreamUtils.ignoreUpstream(Source.future(promise.future)))(Keep.right) + .log("storage-chunkio-write") + .map(_ ⇒ NotUsed) + .recoverWithRetries(1, { case _ ⇒ Source.empty }) + .named("storageWrite") } ) .withAttributes(ActorAttributes.supervisionStrategy(Supervision.resumingDecider)) @@ -125,39 +127,39 @@ private final class ChunkIODispatcher(storageId: StorageId, storageProps: Storag private[this] val readQueue = Source .queue[(ChunkPath, Chunk, Promise[(Chunk, StorageIOResult)])](config.chunkIO.readQueueSize, OverflowStrategy.dropNew) .flatMapMerge( - config.chunkIO.readParallelism, { - case (path, chunk, promise) ⇒ - val repository = subRepository(path.regionId) - val readSource = repository - .read(path.chunkId) // RepositoryStreams.readWithRetries(repository, path.chunkId) - .via(ByteStreams.limit(chunk.checksum.encSize)) - .via(ByteStreams.concat) - .map(ChunkUtils.chunkWithData(chunk, _)) - // .orElse(Source.failed(new IllegalArgumentException("No data read"))) - .named("storageRead") - - val readGraph = GraphDSL.create(readSource) { implicit builder ⇒ readSource ⇒ - import GraphDSL.Implicits._ - val zipResults = builder.add(Zip[Chunk, StorageIOResult]()) - - readSource ~> zipResults.in0 - builder.materializedValue.flatMapConcat(Source.future) ~> zipResults.in1 - - SourceShape(zipResults.out) + config.chunkIO.readParallelism, + { case (path, chunk, promise) ⇒ + val repository = subRepository(path.regionId) + val readSource = repository + .read(path.chunkId) // RepositoryStreams.readWithRetries(repository, path.chunkId) + .via(ByteStreams.limit(chunk.checksum.encSize)) + .via(ByteStreams.concat) + .map(ChunkUtils.chunkWithData(chunk, _)) + // .orElse(Source.failed(new IllegalArgumentException("No data read"))) + .named("storageRead") + + val readGraph = GraphDSL.create(readSource) { implicit builder ⇒ readSource ⇒ + import GraphDSL.Implicits._ + val zipResults = builder.add(Zip[Chunk, StorageIOResult]()) + + readSource ~> zipResults.in0 + builder.materializedValue.flatMapConcat(Source.future) ~> zipResults.in1 + + SourceShape(zipResults.out) + } + + Source + .fromGraph(readGraph) + // .log("storage-read-results") + .alsoToMat(Sink.head)(Keep.right) + .mapMaterializedValue { f ⇒ + promise.completeWith(f) + NotUsed } - - Source - .fromGraph(readGraph) - // .log("storage-read-results") - .alsoToMat(Sink.head)(Keep.right) - .mapMaterializedValue { f ⇒ - promise.completeWith(f) - NotUsed - } - .completionTimeout(config.chunkIO.readTimeout) - .alsoTo(AkkaStreamUtils.failPromiseOnFailure(promise)) - .recoverWithRetries(1, { case _ ⇒ Source.empty }) - .named("storageReadGraph") + .completionTimeout(config.chunkIO.readTimeout) + .alsoTo(AkkaStreamUtils.failPromiseOnFailure(promise)) + .recoverWithRetries(1, { case _ ⇒ Source.empty }) + .named("storageReadGraph") } ) .withAttributes(ActorAttributes.supervisionStrategy(Supervision.resumingDecider)) @@ -207,11 +209,11 @@ private final class ChunkIODispatcher(storageId: StorageId, storageProps: Storag } override def postStop(): Unit = { - chunksWrite.finishAll( - key ⇒ WriteChunk.Failure(key, StorageException.IOFailure(key._1.toStoragePath, new RuntimeException("Chunk IO dispatcher stopped"))) + chunksWrite.finishAll(key ⇒ + WriteChunk.Failure(key, StorageException.IOFailure(key._1.toStoragePath, new RuntimeException("Chunk IO dispatcher stopped"))) ) - chunksRead.finishAll( - key ⇒ ReadChunk.Failure(key, StorageException.IOFailure(key._1.toStoragePath, new RuntimeException("Chunk IO dispatcher stopped"))) + chunksRead.finishAll(key ⇒ + ReadChunk.Failure(key, StorageException.IOFailure(key._1.toStoragePath, new RuntimeException("Chunk IO dispatcher stopped"))) ) writeQueue.complete() readQueue.complete() @@ -282,25 +284,23 @@ private final class ChunkIODispatcher(storageId: StorageId, storageProps: Storag } private[this] def deleteChunks(paths: Set[ChunkPath]): Future[DeleteChunks.Status] = { - val pathsByRegion = paths.groupBy(_.regionId).toVector.map { - case (region, chunks) ⇒ - val localRepository = subRepository(region) - localRepository → chunks + val pathsByRegion = paths.groupBy(_.regionId).toVector.map { case (region, chunks) ⇒ + val localRepository = subRepository(region) + localRepository → chunks } val (ioResult, deleted) = Source(pathsByRegion) - .viaMat(AkkaStreamUtils.flatMapConcatMat { - case (repository, chunks) ⇒ - val deleteSink = Flow[ChunkPath] - .map(_.chunkId) - .toMat(repository.delete)(Keep.right) - - Source(chunks) - .log("chunks-delete") - .alsoToMat(deleteSink)(Keep.right) - .withAttributes( - Attributes.logLevels(onElement = Logging.WarningLevel) and ActorAttributes.supervisionStrategy(Supervision.resumingDecider) - ) + .viaMat(AkkaStreamUtils.flatMapConcatMat { case (repository, chunks) ⇒ + val deleteSink = Flow[ChunkPath] + .map(_.chunkId) + .toMat(repository.delete)(Keep.right) + + Source(chunks) + .log("chunks-delete") + .alsoToMat(deleteSink)(Keep.right) + .withAttributes( + Attributes.logLevels(onElement = Logging.WarningLevel) and ActorAttributes.supervisionStrategy(Supervision.resumingDecider) + ) })(Keep.right) .mapMaterializedValue(_.map(StorageUtils.foldIOResultsIgnoreErrors)) .idleTimeout(sc.config.timeouts.chunksDelete) diff --git a/core/src/main/scala/com/karasiq/shadowcloud/actors/RegionContainer.scala b/core/src/main/scala/com/karasiq/shadowcloud/actors/RegionContainer.scala index 17645f04..aea92efa 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/actors/RegionContainer.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/actors/RegionContainer.scala @@ -21,11 +21,10 @@ private[actors] final class RegionContainer(regionId: RegionId) extends Actor wi private[this] val sc = ShadowCloud() var regionConfig: RegionConfig = sc.configs.regionConfig(regionId) - def receive: Receive = { - case SetConfig(rc) ⇒ - log.info("Region config changed: {}", rc) - this.regionConfig = sc.configs.regionConfig(regionId, rc) - restartActor() + def receive: Receive = { case SetConfig(rc) ⇒ + log.info("Region config changed: {}", rc) + this.regionConfig = sc.configs.regionConfig(regionId, rc) + restartActor() } def startActor(): Unit = { diff --git a/core/src/main/scala/com/karasiq/shadowcloud/actors/RegionDispatcher.scala b/core/src/main/scala/com/karasiq/shadowcloud/actors/RegionDispatcher.scala index 0c634d9b..ac76a141 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/actors/RegionDispatcher.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/actors/RegionDispatcher.scala @@ -102,7 +102,7 @@ private final class RegionDispatcher(regionId: RegionId, regionConfig: RegionCon import sc.implicits.{defaultTimeout, materializer} val storageTracker = StorageTracker() - val chunksTracker = ChunksTracker(regionId, regionConfig, storageTracker, () => schedules.scheduleRetry()) + val chunksTracker = ChunksTracker(regionId, regionConfig, storageTracker, () ⇒ schedules.scheduleRetry()) val indexTracker = RegionIndexTracker(regionId, chunksTracker) private[this] implicit val regionContext = @@ -126,7 +126,7 @@ private final class RegionDispatcher(regionId: RegionId, regionConfig: RegionCon .log("region-grouped-diff") .map(WriteIndexDiff) .withAttributes(ActorAttributes.supervisionStrategy(Supervision.resumingDecider)) - .to(Sink.actorRef(self, Kill, _ => Kill)) + .to(Sink.actorRef(self, Kill, _ ⇒ Kill)) .named("regionPendingQueue") .run() diff --git a/core/src/main/scala/com/karasiq/shadowcloud/actors/RegionGC.scala b/core/src/main/scala/com/karasiq/shadowcloud/actors/RegionGC.scala index ea18dfac..2f6e6632 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/actors/RegionGC.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/actors/RegionGC.scala @@ -7,7 +7,7 @@ import akka.pattern.{ask, pipe} import akka.util.ByteString import com.karasiq.common.memory.MemorySize import com.karasiq.shadowcloud.ShadowCloud -import com.karasiq.shadowcloud.actors.ChunkIODispatcher.{ChunkPath, DeleteChunks => SDeleteChunks} +import com.karasiq.shadowcloud.actors.ChunkIODispatcher.{ChunkPath, DeleteChunks ⇒ SDeleteChunks} import com.karasiq.shadowcloud.actors.RegionIndex.WriteDiff import com.karasiq.shadowcloud.actors.internal.GarbageCollectUtil import com.karasiq.shadowcloud.actors.utils.MessageStatus @@ -47,8 +47,11 @@ object RegionGC { private sealed trait InternalMessage extends Message with PossiblyHarmful private final case class StartGCNow(strategy: GCStrategy = GCStrategy.Default) extends InternalMessage - private final case class GCResult(regionState: RegionGCState, storageStates: Seq[(RegionStorage, StorageGCState, Set[ChunkId])], affectedFiles: Set[File]) - extends InternalMessage + private final case class GCResult( + regionState: RegionGCState, + storageStates: Seq[(RegionStorage, StorageGCState, Set[ChunkId])], + affectedFiles: Set[File] + ) extends InternalMessage // Props private[actors] def props(regionId: RegionId, config: RegionConfig): Props = { @@ -101,7 +104,7 @@ private[actors] final class RegionGC(regionId: RegionId, regionConfig: RegionCon timeoutSchedule = Some(context.system.scheduler.scheduleOnce(sc.config.timeouts.collectGarbage, self, ReceiveTimeout)) context.become(receiveCollecting(Set(sender()), gcStrategy)) - val result = sc.ops.region.synchronize(regionId).flatMap { reports => + val result = sc.ops.region.synchronize(regionId).flatMap { reports ⇒ if (reports.exists(_._2.nonEmpty)) log.warning("Pre-GC sync reports: {}", reports) collectGarbage() } @@ -115,14 +118,13 @@ private[actors] final class RegionGC(regionId: RegionId, regionConfig: RegionCon } def receiveReserve: Receive = { - case Reserve(chunks) => reserved ++= chunks.map(_.withoutData) - case UnReserve(chunks) => reserved --= chunks + case Reserve(chunks) ⇒ reserved ++= chunks.map(_.withoutData) + case UnReserve(chunks) ⇒ reserved --= chunks } private[this] def finishCollecting(receivers: Set[ActorRef], statesFuture: Future[GCResult]): Unit = { - val futureGcResult = statesFuture.map { - case GCResult(regionState, storageStates, affectedFiles) ⇒ - GCReport(regionId, regionState, storageStates.filter(_._2.nonEmpty).map(kv ⇒ kv._1.id → kv._2).toMap, affectedFiles.map(_.withoutChunks)) + val futureGcResult = statesFuture.map { case GCResult(regionState, storageStates, affectedFiles) ⇒ + GCReport(regionId, regionState, storageStates.filter(_._2.nonEmpty).map(kv ⇒ kv._1.id → kv._2).toMap, affectedFiles.map(_.withoutChunks)) } CollectGarbage.wrapFuture(regionId, futureGcResult).foreach { status ⇒ @@ -151,7 +153,13 @@ private[actors] final class RegionGC(regionId: RegionId, regionConfig: RegionCon case GCResult(regionState, storageStates, affectedFiles) ⇒ val level = if (affectedFiles.nonEmpty) Logging.WarningLevel else Logging.InfoLevel - log.log(level, "Region GC states = {}, storages = {}, affected files = {}", regionState, storageStates.map(kv => kv._1.id -> kv._2).toMap, affectedFiles.map(_.path)) + log.log( + level, + "Region GC states = {}, storages = {}, affected files = {}", + regionState, + storageStates.map(kv ⇒ kv._1.id → kv._2).toMap, + affectedFiles.map(_.path) + ) val delete = gcStrategy match { case GCStrategy.Default ⇒ gcConfig.autoDelete @@ -159,10 +167,9 @@ private[actors] final class RegionGC(regionId: RegionId, regionConfig: RegionCon case GCStrategy.Delete ⇒ true } val patchedRegionState = regionState.copy(orphanedChunks = regionState.orphanedChunks -- reserved) - val patchedStorageStates = storageStates.map { - case (st, ss, chunks) => - val chunkKey = ChunkUtils.getChunkKeyMapper(regionConfig, st.config) - (st, ss.copy(ss.notIndexed -- reserved.map(chunkKey), ss.notExisting), chunks) + val patchedStorageStates = storageStates.map { case (st, ss, chunks) ⇒ + val chunkKey = ChunkUtils.getChunkKeyMapper(regionConfig, st.config) + (st, ss.copy(ss.notIndexed -- reserved.map(chunkKey), ss.notExisting), chunks) } val future = deleteGarbage(patchedRegionState, patchedStorageStates, delete) .map(_ ⇒ GCResult(patchedRegionState, patchedStorageStates, affectedFiles)) @@ -206,8 +213,8 @@ private[actors] final class RegionGC(regionId: RegionId, regionConfig: RegionCon def affectedFiles(files: FolderIndex, storageStates: Seq[StorageGCState]): Set[File] = { val chunks = (for { - st <- storageStates - chunk <- st.notExisting + st ← storageStates + chunk ← st.notExisting } yield chunk).toSet files.filesIterator.filter(_.chunks.exists(chunks)).toSet @@ -255,28 +262,25 @@ private[actors] final class RegionGC(regionId: RegionId, regionConfig: RegionCon } def deleteFromStorages(chunks: Seq[(RegionStorage, Set[ByteString])]): Future[(Set[ByteString], StorageIOResult)] = { - val futures = chunks.map { - case (storage, chunks) ⇒ - val paths = chunks.map(ChunkPath(regionId, _)) - if (log.isDebugEnabled) log.debug("Deleting chunks from storage {}: [{}]", storage.id, Utils.printHashes(chunks)) - SDeleteChunks.unwrapFuture((storage.dispatcher ? SDeleteChunks(paths))(sc.config.timeouts.chunksDelete)) + val futures = chunks.map { case (storage, chunks) ⇒ + val paths = chunks.map(ChunkPath(regionId, _)) + if (log.isDebugEnabled) log.debug("Deleting chunks from storage {}: [{}]", storage.id, Utils.printHashes(chunks)) + SDeleteChunks.unwrapFuture((storage.dispatcher ? SDeleteChunks(paths))(sc.config.timeouts.chunksDelete)) } - Future.foldLeft(futures.toVector)((Set.empty[ByteString], StorageIOResult.empty)) { - case ((deleted, result), (deleted1, result1)) ⇒ - (deleted ++ deleted1.map(_.chunkId), StorageUtils.foldIOResultsIgnoreErrors(result, result1)) + Future.foldLeft(futures.toVector)((Set.empty[ByteString], StorageIOResult.empty)) { case ((deleted, result), (deleted1, result1)) ⇒ + (deleted ++ deleted1.map(_.chunkId), StorageUtils.foldIOResultsIgnoreErrors(result, result1)) } } def deleteFromIndexes(chunks: Seq[(RegionStorage, Set[Chunk])]): Future[Done] = { - val futures = chunks.map { - case (storage, chunks) ⇒ - import sc.implicits.defaultTimeout - if (log.isDebugEnabled) log.debug("Deleting chunks from index {}: [{}]", storage.id, Utils.printChunkHashes(chunks)) - WriteDiff.unwrapFuture( - storage.dispatcher ? - StorageIndex.Envelope(regionId, WriteDiff(IndexDiff.deleteChunks(chunks.toSeq: _*))) - ) + val futures = chunks.map { case (storage, chunks) ⇒ + import sc.implicits.defaultTimeout + if (log.isDebugEnabled) log.debug("Deleting chunks from index {}: [{}]", storage.id, Utils.printChunkHashes(chunks)) + WriteDiff.unwrapFuture( + storage.dispatcher ? + StorageIndex.Envelope(regionId, WriteDiff(IndexDiff.deleteChunks(chunks.toSeq: _*))) + ) } Future.foldLeft(futures.toList)(Done)((_, _) ⇒ Done) } diff --git a/core/src/main/scala/com/karasiq/shadowcloud/actors/RegionIndex.scala b/core/src/main/scala/com/karasiq/shadowcloud/actors/RegionIndex.scala index 50f2836c..ced924b3 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/actors/RegionIndex.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/actors/RegionIndex.scala @@ -141,14 +141,13 @@ private[actors] final class RegionIndex(storageId: StorageId, regionId: RegionId // ----------------------------------------------------------------------- // Idle state // ----------------------------------------------------------------------- - def receiveWait: Receive = { - case Synchronize ⇒ - log.debug("Starting synchronization") + def receiveWait: Receive = { case Synchronize ⇒ + log.debug("Starting synchronization") - if (sender() != self && sender() != Actor.noSender) - state.pendingSync.addWaiter(state.indexId, sender()) + if (sender() != self && sender() != Actor.noSender) + state.pendingSync.addWaiter(state.indexId, sender()) - deferAsync(NotUsed)(_ ⇒ synchronization.start()) + deferAsync(NotUsed)(_ ⇒ synchronization.start()) } // ----------------------------------------------------------------------- @@ -335,9 +334,8 @@ private[actors] final class RegionIndex(storageId: StorageId, regionId: RegionId Source .single((newSequenceNr, diff)) - .alsoTo(Sink.foreach { - case (sequenceNr, diff) ⇒ - log.info("Writing diff #{}: {}", sequenceNr, diff) + .alsoTo(Sink.foreach { case (sequenceNr, diff) ⇒ + log.info("Writing diff #{}: {}", sequenceNr, diff) }) .via(toIndexDataWithKey) .via(state.streams.write(repository)) @@ -413,22 +411,21 @@ private[actors] final class RegionIndex(storageId: StorageId, regionId: RegionId .log("compact-write") } - writeResult.flatMapConcat( - wr ⇒ - wr.ioResult match { - case StorageIOResult.Success(_, _) ⇒ - Source(diffsToDelete) - .via(state.streams.delete(repository)) - .log("compact-delete") - .withAttributes(ActorAttributes.logLevels(onElement = Logging.InfoLevel)) - .filter(_.ioResult.isSuccess) - .map(_.key) - .fold(Set.empty[SequenceNr])(_ + _) - .map(CompactSuccess(_, Some(wr).filterNot(_.diff.isEmpty))) - - case StorageIOResult.Failure(_, error) ⇒ - Source.failed(error) - } + writeResult.flatMapConcat(wr ⇒ + wr.ioResult match { + case StorageIOResult.Success(_, _) ⇒ + Source(diffsToDelete) + .via(state.streams.delete(repository)) + .log("compact-delete") + .withAttributes(ActorAttributes.logLevels(onElement = Logging.InfoLevel)) + .filter(_.ioResult.isSuccess) + .map(_.key) + .fold(Set.empty[SequenceNr])(_ + _) + .map(CompactSuccess(_, Some(wr).filterNot(_.diff.isEmpty))) + + case StorageIOResult.Failure(_, error) ⇒ + Source.failed(error) + } ) } diff --git a/core/src/main/scala/com/karasiq/shadowcloud/actors/RegionSupervisor.scala b/core/src/main/scala/com/karasiq/shadowcloud/actors/RegionSupervisor.scala index 15fc0eaa..17f9333e 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/actors/RegionSupervisor.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/actors/RegionSupervisor.scala @@ -279,14 +279,12 @@ private sealed trait RegionSupervisorState { self: RegionSupervisor ⇒ def loadState(storages: collection.Map[String, StorageSnapshot], regions: collection.Map[String, RegionSnapshot]): Unit = { state.clear() - storages.foreach { - case (storageId, StorageSnapshot(props, active)) ⇒ - updateState(StorageAdded(storageId, props, active)) + storages.foreach { case (storageId, StorageSnapshot(props, active)) ⇒ + updateState(StorageAdded(storageId, props, active)) } - regions.foreach { - case (regionId, RegionSnapshot(regionConfig, storages, active)) ⇒ - updateState(RegionAdded(regionId, regionConfig, active)) - storages.foreach(storageId ⇒ updateState(StorageRegistered(regionId, storageId))) + regions.foreach { case (regionId, RegionSnapshot(regionConfig, storages, active)) ⇒ + updateState(RegionAdded(regionId, regionConfig, active)) + storages.foreach(storageId ⇒ updateState(StorageRegistered(regionId, storageId))) } } } diff --git a/core/src/main/scala/com/karasiq/shadowcloud/actors/StorageContainer.scala b/core/src/main/scala/com/karasiq/shadowcloud/actors/StorageContainer.scala index 52815b48..8820bcbb 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/actors/StorageContainer.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/actors/StorageContainer.scala @@ -29,11 +29,10 @@ private[actors] final class StorageContainer(instantiator: StorageInstantiator, private[this] var storageProps: StorageProps = StorageProps.inMemory - def receive: Receive = { - case SetProps(props) ⇒ - log.info("Storage props changed: {}", props) - this.storageProps = props - restartActor() + def receive: Receive = { case SetProps(props) ⇒ + log.info("Storage props changed: {}", props) + this.storageProps = props + restartActor() } def startActor(): Unit = { diff --git a/core/src/main/scala/com/karasiq/shadowcloud/actors/StorageDispatcher.scala b/core/src/main/scala/com/karasiq/shadowcloud/actors/StorageDispatcher.scala index fbf9644c..3076f679 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/actors/StorageDispatcher.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/actors/StorageDispatcher.scala @@ -78,9 +78,8 @@ private final class StorageDispatcher( .queue[(ChunkPath, Chunk)](sc.config.queues.chunksIndex, OverflowStrategy.dropNew) .via(AkkaStreamUtils.groupedOrInstant(sc.config.queues.chunksIndex, sc.config.queues.chunksIndexTime)) .filter(_.nonEmpty) - .mapConcat(_.groupBy(_._1.regionId).map { - case (regionId, chunks) ⇒ - StorageIndex.Envelope(regionId, RegionIndex.WriteDiff(IndexDiff.newChunks(chunks.map(_._2): _*))) + .mapConcat(_.groupBy(_._1.regionId).map { case (regionId, chunks) ⇒ + StorageIndex.Envelope(regionId, RegionIndex.WriteDiff(IndexDiff.newChunks(chunks.map(_._2): _*))) }) .withAttributes(ActorAttributes.supervisionStrategy(Supervision.resumingDecider)) .to(Sink.actorRef(index, Status.Failure(new IllegalStateException("Queue stopped")))) diff --git a/core/src/main/scala/com/karasiq/shadowcloud/actors/StorageIndex.scala b/core/src/main/scala/com/karasiq/shadowcloud/actors/StorageIndex.scala index 63217620..b8a2bb6f 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/actors/StorageIndex.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/actors/StorageIndex.scala @@ -12,7 +12,7 @@ import com.karasiq.shadowcloud.storage.props.StorageProps import com.karasiq.shadowcloud.storage.repository.CategorizedRepository import com.karasiq.shadowcloud.storage.utils.{IndexMerger, StorageUtils} -import scala.collection.mutable.{AnyRefMap => MMap} +import scala.collection.mutable.{AnyRefMap ⇒ MMap} import scala.concurrent.duration._ import scala.concurrent.{ExecutionContext, Future} @@ -59,9 +59,8 @@ private[actors] final class StorageIndex(storageId: StorageId, storageProps: Sto case SynchronizeAll ⇒ log.debug("Synchronizing all indexes") - val futures = subIndexes.map { - case (regionId, dispatcher) ⇒ - RegionIndex.Synchronize.unwrapFuture(dispatcher ? RegionIndex.Synchronize).map((regionId, _)) + val futures = subIndexes.map { case (regionId, dispatcher) ⇒ + RegionIndex.Synchronize.unwrapFuture(dispatcher ? RegionIndex.Synchronize).map((regionId, _)) } val result = Future.sequence(futures.toVector).map(_.toMap) SynchronizeAll.wrapFuture(storageId, result).pipeTo(sender()) @@ -75,10 +74,9 @@ private[actors] final class StorageIndex(storageId: StorageId, storageProps: Sto stopRegionDispatcher(regionId, clear) case GetIndexes ⇒ - val futures = subIndexes.map { - case (regionId, dispatcher) ⇒ - val future = RegionIndex.GetIndex.unwrapFuture(dispatcher ? RegionIndex.GetIndex) - future.map((regionId, _)) + val futures = subIndexes.map { case (regionId, dispatcher) ⇒ + val future = RegionIndex.GetIndex.unwrapFuture(dispatcher ? RegionIndex.GetIndex) + future.map((regionId, _)) } Future .sequence(futures) diff --git a/core/src/main/scala/com/karasiq/shadowcloud/actors/events/RegionEvents.scala b/core/src/main/scala/com/karasiq/shadowcloud/actors/events/RegionEvents.scala index f5553850..7b82526e 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/actors/events/RegionEvents.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/actors/events/RegionEvents.scala @@ -8,6 +8,6 @@ object RegionEvents { // Events sealed trait Event case class IndexUpdated(sequenceNr: RegionKey, diff: IndexDiff) extends Event - case class IndexDeleted(keys: Set[RegionKey]) extends Event - case class ChunkWritten(storageId: StorageId, chunk: Chunk) extends Event + case class IndexDeleted(keys: Set[RegionKey]) extends Event + case class ChunkWritten(storageId: StorageId, chunk: Chunk) extends Event } diff --git a/core/src/main/scala/com/karasiq/shadowcloud/actors/events/StorageEvents.scala b/core/src/main/scala/com/karasiq/shadowcloud/actors/events/StorageEvents.scala index 820c6dbf..eb06358d 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/actors/events/StorageEvents.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/actors/events/StorageEvents.scala @@ -1,7 +1,5 @@ package com.karasiq.shadowcloud.actors.events - - import com.karasiq.shadowcloud.actors.ChunkIODispatcher.ChunkPath import com.karasiq.shadowcloud.index.diffs.IndexDiff import com.karasiq.shadowcloud.model.utils.StorageHealth @@ -10,12 +8,10 @@ import com.karasiq.shadowcloud.storage.utils.IndexMerger object StorageEvents { sealed trait Event - case class IndexLoaded(regionId: RegionId, state: IndexMerger.State[Long]) extends Event - case class PendingIndexUpdated(regionId: RegionId, diff: IndexDiff) extends Event + case class IndexLoaded(regionId: RegionId, state: IndexMerger.State[Long]) extends Event + case class PendingIndexUpdated(regionId: RegionId, diff: IndexDiff) extends Event case class IndexUpdated(regionId: RegionId, sequenceNr: SequenceNr, diff: IndexDiff, remote: Boolean) extends Event - case class IndexDeleted(regionId: RegionId, sequenceNrs: Set[Long]) extends Event - case class ChunkWritten(path: ChunkPath, chunk: Chunk) extends Event - case class HealthUpdated(health: StorageHealth) extends Event + case class IndexDeleted(regionId: RegionId, sequenceNrs: Set[Long]) extends Event + case class ChunkWritten(path: ChunkPath, chunk: Chunk) extends Event + case class HealthUpdated(health: StorageHealth) extends Event } - - diff --git a/core/src/main/scala/com/karasiq/shadowcloud/actors/events/SupervisorEvents.scala b/core/src/main/scala/com/karasiq/shadowcloud/actors/events/SupervisorEvents.scala index a585d831..9aed3314 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/actors/events/SupervisorEvents.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/actors/events/SupervisorEvents.scala @@ -7,11 +7,11 @@ import com.karasiq.shadowcloud.storage.props.StorageProps object SupervisorEvents { sealed trait Event final case class RegionAdded(regionId: RegionId, regionConfig: RegionConfig, active: Boolean) extends Event - final case class RegionDeleted(regionId: RegionId) extends Event - final case class StorageAdded(storageId: StorageId, props: StorageProps, active: Boolean) extends Event - final case class StorageDeleted(storageId: StorageId) extends Event - final case class StorageRegistered(regionId: RegionId, storageId: StorageId) extends Event - final case class StorageUnregistered(regionId: RegionId, storageId: StorageId) extends Event - final case class StorageStateChanged(storageId: StorageId, active: Boolean) extends Event - final case class RegionStateChanged(regionId: RegionId, active: Boolean) extends Event + final case class RegionDeleted(regionId: RegionId) extends Event + final case class StorageAdded(storageId: StorageId, props: StorageProps, active: Boolean) extends Event + final case class StorageDeleted(storageId: StorageId) extends Event + final case class StorageRegistered(regionId: RegionId, storageId: StorageId) extends Event + final case class StorageUnregistered(regionId: RegionId, storageId: StorageId) extends Event + final case class StorageStateChanged(storageId: StorageId, active: Boolean) extends Event + final case class RegionStateChanged(regionId: RegionId, active: Boolean) extends Event } diff --git a/core/src/main/scala/com/karasiq/shadowcloud/actors/internal/ChunksTracker.scala b/core/src/main/scala/com/karasiq/shadowcloud/actors/internal/ChunksTracker.scala index 56b129a7..e2d51388 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/actors/internal/ChunksTracker.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/actors/internal/ChunksTracker.scala @@ -21,10 +21,9 @@ import com.karasiq.shadowcloud.storage.replication.ChunkStatusProvider.{ChunkSta import com.karasiq.shadowcloud.storage.replication.RegionStorageProvider.RegionStorage import com.karasiq.shadowcloud.utils.{ChunkUtils, Utils} - private[actors] object ChunksTracker { - def apply(regionId: RegionId, config: RegionConfig, storageTracker: StorageTracker, scheduleRetry: () ⇒ Unit)( - implicit context: ActorContext, + def apply(regionId: RegionId, config: RegionConfig, storageTracker: StorageTracker, scheduleRetry: () ⇒ Unit)(implicit + context: ActorContext, sc: ShadowCloudExtension ): ChunksTracker = { new ChunksTracker(regionId, config, storageTracker, scheduleRetry) @@ -34,8 +33,8 @@ private[actors] object ChunksTracker { } // Internal region logic -private[actors] final class ChunksTracker(regionId: RegionId, config: RegionConfig, storageTracker: StorageTracker, scheduleRetry: () ⇒ Unit)( - implicit context: ActorContext, +private[actors] final class ChunksTracker(regionId: RegionId, config: RegionConfig, storageTracker: StorageTracker, scheduleRetry: () ⇒ Unit)(implicit + context: ActorContext, sc: ShadowCloudExtension ) { import context.dispatcher @@ -151,8 +150,8 @@ private[actors] final class ChunksTracker(regionId: RegionId, config: RegionConf } } - def repairChunk(chunk: Chunk, newAffinity: Option[ChunkWriteAffinity], receiver: ActorRef)( - implicit storageSelector: StorageSelector + def repairChunk(chunk: Chunk, newAffinity: Option[ChunkWriteAffinity], receiver: ActorRef)(implicit + storageSelector: StorageSelector ): Option[ChunkStatus] = { val available = storageSelector.available(chunk.checksum.encSize).map(_.id).toSet @@ -288,12 +287,11 @@ private[actors] final class ChunksTracker(regionId: RegionId, config: RegionConf status } else { log.debug("Writing chunk to {}: {}", writtenStorages, status.chunk) - writes.foreach { - case (storage, future) ⇒ - future - .map(ChunkWriteSuccess(storage.id, _)) - .recover { case error ⇒ ChunkWriteFailed(storage.id, status.chunk, error) } - .pipeTo(context.self) + writes.foreach { case (storage, future) ⇒ + future + .map(ChunkWriteSuccess(storage.id, _)) + .recover { case error ⇒ ChunkWriteFailed(storage.id, status.chunk, error) } + .pipeTo(context.self) } storages.state.markAsWriting(status, writtenStorages.map(_.id): _*) } diff --git a/core/src/main/scala/com/karasiq/shadowcloud/actors/internal/GarbageCollectUtil.scala b/core/src/main/scala/com/karasiq/shadowcloud/actors/internal/GarbageCollectUtil.scala index 0ffa8b7a..64e8087a 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/actors/internal/GarbageCollectUtil.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/actors/internal/GarbageCollectUtil.scala @@ -20,7 +20,7 @@ private[actors] final class GarbageCollectUtil(regionConfig: RegionConfig) { def checkStorage(index: IndexMerger[_], storageConfig: StorageConfig, storageChunks: Set[ByteString]): StorageGCState = { val indexPersistedChunks = index.chunks - val indexPendingChunks = index.chunks.patch(index.pending.chunks) + val indexPendingChunks = index.chunks.patch(index.pending.chunks) StorageGCState( notIndexedChunks(indexPendingChunks, storageConfig, storageChunks), @@ -30,7 +30,7 @@ private[actors] final class GarbageCollectUtil(regionConfig: RegionConfig) { def checkRegion(index: IndexMerger[_]): RegionGCState = { val indexPersistedChunks = index.chunks - val indexPendingFolders = index.foldersWithPending + val indexPendingFolders = index.foldersWithPending RegionGCState( orphanedChunks(indexPersistedChunks, indexPendingFolders), @@ -50,7 +50,7 @@ private[actors] final class GarbageCollectUtil(regionConfig: RegionConfig) { val filesToDelete = filesByPath.flatMap { case (_, files) ⇒ def isFileRecent(file: File): Boolean = { - val now = Utils.timestamp + val now = Utils.timestamp val retention = gcConfig.keepRecentFiles.toMillis (file.timestamp.lastModified + retention) >= now } diff --git a/core/src/main/scala/com/karasiq/shadowcloud/actors/internal/RegionIndexTracker.scala b/core/src/main/scala/com/karasiq/shadowcloud/actors/internal/RegionIndexTracker.scala index 6c8d43fb..b3c0f46a 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/actors/internal/RegionIndexTracker.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/actors/internal/RegionIndexTracker.scala @@ -21,21 +21,19 @@ import com.karasiq.shadowcloud.storage.utils.IndexMerger.RegionKey import com.karasiq.shadowcloud.utils.Utils private[actors] object RegionIndexTracker { - def apply(regionId: RegionId, chunksTracker: ChunksTracker) - (implicit sc: ShadowCloudExtension): RegionIndexTracker = { + def apply(regionId: RegionId, chunksTracker: ChunksTracker)(implicit sc: ShadowCloudExtension): RegionIndexTracker = { new RegionIndexTracker(regionId, chunksTracker) } } -private[actors] final class RegionIndexTracker(regionId: RegionId, chunksTracker: ChunksTracker) - (implicit sc: ShadowCloudExtension) { +private[actors] final class RegionIndexTracker(regionId: RegionId, chunksTracker: ChunksTracker)(implicit sc: ShadowCloudExtension) { import sc.implicits.{defaultTimeout, executionContext} // ----------------------------------------------------------------------- // Context // ----------------------------------------------------------------------- private[this] val log = Logging(sc.implicits.actorSystem, s"$regionId-index") - val globalIndex = IndexMerger.region() + val globalIndex = IndexMerger.region() // ----------------------------------------------------------------------- // Storages @@ -43,8 +41,9 @@ private[actors] final class RegionIndexTracker(regionId: RegionId, chunksTracker object storages { object state { def extractDiffs(storageId: StorageId): Seq[(SequenceNr, IndexDiff)] = { - val diffs = for ((RegionKey(_, `storageId`, sequenceNr), diff) ← globalIndex.diffs) - yield (sequenceNr, diff) + val diffs = + for ((RegionKey(_, `storageId`, sequenceNr), diff) ← globalIndex.diffs) + yield (sequenceNr, diff) diffs.toVector } @@ -90,8 +89,10 @@ private[actors] final class RegionIndexTracker(regionId: RegionId, chunksTracker } def getIndex(storage: RegionStorage): Future[IndexMerger.State[SequenceNr]] = { - RegionIndex.GetIndex.unwrapFuture(storage.dispatcher ? - StorageIndex.Envelope(regionId, RegionIndex.GetIndex)) + RegionIndex.GetIndex.unwrapFuture( + storage.dispatcher ? + StorageIndex.Envelope(regionId, RegionIndex.GetIndex) + ) } def writeIndex(storage: RegionStorage, diff: IndexDiff): Unit = { @@ -106,8 +107,7 @@ private[actors] final class RegionIndexTracker(regionId: RegionId, chunksTracker log.warning("No index storages available on {}", regionId) } else { if (log.isDebugEnabled) { - log.debug("Writing to virtual region [{}] index: {} (storages = [{}])", - regionId, diff, Utils.printValues(storages.map(_.id))) + log.debug("Writing to virtual region [{}] index: {} (storages = [{}])", regionId, diff, Utils.printValues(storages.map(_.id))) } storages.foreach(_.dispatcher ! StorageIndex.Envelope(regionId, WriteDiff(diff))) @@ -163,7 +163,8 @@ private[actors] final class RegionIndexTracker(regionId: RegionId, chunksTracker def getFileAvailability(file: File): FileAvailability = { val chunkStoragePairs = file.chunks.flatMap { chunk ⇒ - chunksTracker.chunks.getChunkStatus(chunk) + chunksTracker.chunks + .getChunkStatus(chunk) .toSeq .flatMap(_.availability.hasChunk) .map(_ → chunk) diff --git a/core/src/main/scala/com/karasiq/shadowcloud/actors/internal/RegionTracker.scala b/core/src/main/scala/com/karasiq/shadowcloud/actors/internal/RegionTracker.scala index 1d1487cc..6c46596b 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/actors/internal/RegionTracker.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/actors/internal/RegionTracker.scala @@ -2,7 +2,7 @@ package com.karasiq.shadowcloud.actors.internal import akka.actor.ActorContext import com.karasiq.shadowcloud.ShadowCloud -import com.karasiq.shadowcloud.actors.utils.{ActorState => State} +import com.karasiq.shadowcloud.actors.utils.{ActorState ⇒ State} import com.karasiq.shadowcloud.actors.{RegionContainer, RegionDispatcher, StorageContainer, StorageIndex} import com.karasiq.shadowcloud.config.RegionConfig import com.karasiq.shadowcloud.model.utils.StorageHealth @@ -157,16 +157,18 @@ private[actors] final class RegionTracker(implicit context: ActorContext) { // ----------------------------------------------------------------------- def deleteRegion(regionId: RegionId): RegionStatus = { require(containsRegion(regionId)) - storages.foreach { - case (storageId, storage) ⇒ - if (storage.regions.contains(regionId)) - storages += storageId → storage.copy(regions = storage.regions - regionId) + storages.foreach { case (storageId, storage) ⇒ + if (storage.regions.contains(regionId)) + storages += storageId → storage.copy(regions = storage.regions - regionId) } val status = regions.remove(regionId).get - State.ifActive(status.actorState, { dispatcher ⇒ - context.unwatch(dispatcher) - context.stop(dispatcher) - }) + State.ifActive( + status.actorState, + { dispatcher ⇒ + context.unwatch(dispatcher) + context.stop(dispatcher) + } + ) status.storages.flatMap(storages.get).foreach { storage ⇒ State.ifActive(storage.actorState, _ ! StorageIndex.CloseIndex(regionId, clear = true)) } @@ -175,26 +177,33 @@ private[actors] final class RegionTracker(implicit context: ActorContext) { def deleteStorage(storageId: StorageId): StorageStatus = { require(containsStorage(storageId)) - regions.foreach { - case (regionId, region) ⇒ - if (region.storages.contains(storageId)) { - regions += regionId → region.copy(storages = region.storages - storageId) - State.ifActive(region.actorState, _ ! RegionDispatcher.DetachStorage(storageId)) - } + regions.foreach { case (regionId, region) ⇒ + if (region.storages.contains(storageId)) { + regions += regionId → region.copy(storages = region.storages - storageId) + State.ifActive(region.actorState, _ ! RegionDispatcher.DetachStorage(storageId)) + } } val status = storages.remove(storageId).get - State.ifActive(status.actorState, { dispatcher ⇒ - context.unwatch(dispatcher) - context.stop(dispatcher) - }) + State.ifActive( + status.actorState, + { dispatcher ⇒ + context.unwatch(dispatcher) + context.stop(dispatcher) + } + ) status } def clear(): Unit = { - (regions.values.map(_.actorState) ++ storages.values.map(_.actorState)).foreach(State.ifActive(_, { dispatcher ⇒ - context.unwatch(dispatcher) - context.stop(dispatcher) - })) + (regions.values.map(_.actorState) ++ storages.values.map(_.actorState)).foreach( + State.ifActive( + _, + { dispatcher ⇒ + context.unwatch(dispatcher) + context.stop(dispatcher) + } + ) + ) regions.clear() storages.clear() } diff --git a/core/src/main/scala/com/karasiq/shadowcloud/actors/internal/StorageInstantiator.scala b/core/src/main/scala/com/karasiq/shadowcloud/actors/internal/StorageInstantiator.scala index a2211841..1e34b552 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/actors/internal/StorageInstantiator.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/actors/internal/StorageInstantiator.scala @@ -1,7 +1,5 @@ package com.karasiq.shadowcloud.actors.internal - - import akka.actor.{ActorContext, ActorRef} import com.karasiq.shadowcloud.model.StorageId import com.karasiq.shadowcloud.providers.SCModules diff --git a/core/src/main/scala/com/karasiq/shadowcloud/actors/internal/StorageTracker.scala b/core/src/main/scala/com/karasiq/shadowcloud/actors/internal/StorageTracker.scala index 30b9ecd2..5da58f48 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/actors/internal/StorageTracker.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/actors/internal/StorageTracker.scala @@ -21,7 +21,7 @@ private[actors] final class StorageTracker(implicit context: ActorContext) exten // ----------------------------------------------------------------------- // State // ----------------------------------------------------------------------- - private[this] val sc = ShadowCloud() + private[this] val sc = ShadowCloud() private[this] val storagesById = mutable.AnyRefMap[String, RegionStorage]() private[this] val storagesByAR = mutable.AnyRefMap[ActorRef, RegionStorage]() @@ -42,7 +42,7 @@ private[actors] final class StorageTracker(implicit context: ActorContext) exten def register(storageId: StorageId, props: StorageProps, dispatcher: ActorRef, health: StorageHealth): Unit = { context.watch(dispatcher) val storage = RegionStorage(storageId, props, sc.configs.storageConfig(storageId, props), dispatcher, health) - storagesById += storageId → storage + storagesById += storageId → storage storagesByAR += dispatcher → storage sc.eventStreams.storage.subscribe(context.self, storageId) } @@ -84,7 +84,7 @@ private[actors] final class StorageTracker(implicit context: ActorContext) exten def updateHealth(storageId: StorageId, health: StorageHealth): Unit = { storagesById.get(storageId).foreach { storage ⇒ val newStatus = storage.copy(health = health) - storagesById += storageId → newStatus + storagesById += storageId → newStatus storagesByAR += storage.dispatcher → newStatus } } diff --git a/core/src/main/scala/com/karasiq/shadowcloud/actors/utils/PendingOperations.scala b/core/src/main/scala/com/karasiq/shadowcloud/actors/utils/PendingOperations.scala index 49f60c93..07c9d374 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/actors/utils/PendingOperations.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/actors/utils/PendingOperations.scala @@ -4,6 +4,6 @@ import com.karasiq.shadowcloud.actors.ChunkIODispatcher.ChunkPath import com.karasiq.shadowcloud.model.Chunk private[actors] object PendingOperations { - def withChunk: PendingOperation[Chunk] = new PendingOperation + def withChunk: PendingOperation[Chunk] = new PendingOperation def withRegionChunk: PendingOperation[(ChunkPath, Chunk)] = new PendingOperation } diff --git a/core/src/main/scala/com/karasiq/shadowcloud/config/EncryptionConfig.scala b/core/src/main/scala/com/karasiq/shadowcloud/config/EncryptionConfig.scala index e470b673..5c052dab 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/config/EncryptionConfig.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/config/EncryptionConfig.scala @@ -1,14 +1,16 @@ package com.karasiq.shadowcloud.config - - import com.karasiq.common.configs.ConfigImplicits import com.karasiq.shadowcloud.model.crypto.EncryptionMethod import com.typesafe.config.{Config, ConfigException} -private[shadowcloud] case class EncryptionConfig(rootConfig: Config, chunks: EncryptionMethod, - index: EncryptionMethod, keys: EncryptionMethod, - maxKeyReuse: Int) extends WrappedConfig +private[shadowcloud] case class EncryptionConfig( + rootConfig: Config, + chunks: EncryptionMethod, + index: EncryptionMethod, + keys: EncryptionMethod, + maxKeyReuse: Int +) extends WrappedConfig private[shadowcloud] object EncryptionConfig extends WrappedConfigFactory[EncryptionConfig] with ConfigImplicits { def apply(config: Config): EncryptionConfig = { @@ -24,9 +26,10 @@ private[shadowcloud] object EncryptionConfig extends WrappedConfigFactory[Encryp private[this] def getEncryptionMethod(config: Config, path: String): EncryptionMethod = { try { CryptoProps.encryption(config.getConfigOrRef(path)) - } catch { case _: ConfigException.Missing ⇒ - val alg = config.getString(path) - EncryptionMethod(alg) + } catch { + case _: ConfigException.Missing ⇒ + val alg = config.getString(path) + EncryptionMethod(alg) } } } diff --git a/core/src/main/scala/com/karasiq/shadowcloud/config/GCConfig.scala b/core/src/main/scala/com/karasiq/shadowcloud/config/GCConfig.scala index 22bf60c2..cd12c15c 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/config/GCConfig.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/config/GCConfig.scala @@ -5,8 +5,8 @@ import com.typesafe.config.Config import scala.concurrent.duration.{FiniteDuration, _} -case class GCConfig(rootConfig: Config, runOnLowSpace: Option[Long], autoDelete: Boolean, - keepFileRevisions: Int, keepRecentFiles: FiniteDuration) extends WrappedConfig +case class GCConfig(rootConfig: Config, runOnLowSpace: Option[Long], autoDelete: Boolean, keepFileRevisions: Int, keepRecentFiles: FiniteDuration) + extends WrappedConfig object GCConfig extends WrappedConfigFactory[GCConfig] with ConfigImplicits { def apply(config: Config): GCConfig = { diff --git a/core/src/main/scala/com/karasiq/shadowcloud/config/HashingConfig.scala b/core/src/main/scala/com/karasiq/shadowcloud/config/HashingConfig.scala index aa7db276..9c58808a 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/config/HashingConfig.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/config/HashingConfig.scala @@ -1,14 +1,16 @@ package com.karasiq.shadowcloud.config - - import com.karasiq.common.configs.ConfigImplicits import com.karasiq.shadowcloud.model.crypto.HashingMethod import com.typesafe.config.{Config, ConfigException} -private[shadowcloud] case class HashingConfig(rootConfig: Config, chunks: HashingMethod, - chunksEncrypted: HashingMethod, files: HashingMethod, - filesEncrypted: HashingMethod) extends WrappedConfig +private[shadowcloud] case class HashingConfig( + rootConfig: Config, + chunks: HashingMethod, + chunksEncrypted: HashingMethod, + files: HashingMethod, + filesEncrypted: HashingMethod +) extends WrappedConfig private[shadowcloud] object HashingConfig extends WrappedConfigFactory[HashingConfig] with ConfigImplicits { def apply(config: Config): HashingConfig = { @@ -24,9 +26,10 @@ private[shadowcloud] object HashingConfig extends WrappedConfigFactory[HashingCo private[this] def getHashingMethod(config: Config, path: String): HashingMethod = { try { CryptoProps.hashing(config.getConfigOrRef(path)) - } catch { case _: ConfigException.Missing ⇒ - val alg = config.getString(path) - HashingMethod(alg) + } catch { + case _: ConfigException.Missing ⇒ + val alg = config.getString(path) + HashingMethod(alg) } } } diff --git a/core/src/main/scala/com/karasiq/shadowcloud/config/MetadataConfig.scala b/core/src/main/scala/com/karasiq/shadowcloud/config/MetadataConfig.scala index 3e83c722..716128fb 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/config/MetadataConfig.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/config/MetadataConfig.scala @@ -5,8 +5,8 @@ import com.typesafe.config.Config import com.karasiq.common.configs.ConfigImplicits import com.karasiq.shadowcloud.metadata.MetadataProvider -case class MetadataConfig(rootConfig: Config, fileSizeLimit: Long, - mimeProbeSize: Int, providers: ProvidersConfig[MetadataProvider]) extends WrappedConfig +case class MetadataConfig(rootConfig: Config, fileSizeLimit: Long, mimeProbeSize: Int, providers: ProvidersConfig[MetadataProvider]) + extends WrappedConfig object MetadataConfig extends WrappedConfigFactory[MetadataConfig] with ConfigImplicits { def apply(config: Config): MetadataConfig = { @@ -17,4 +17,4 @@ object MetadataConfig extends WrappedConfigFactory[MetadataConfig] with ConfigIm ProvidersConfig.withType[MetadataProvider](config.getConfig("providers")) ) } -} \ No newline at end of file +} diff --git a/core/src/main/scala/com/karasiq/shadowcloud/config/PersistenceConfig.scala b/core/src/main/scala/com/karasiq/shadowcloud/config/PersistenceConfig.scala index a08adb0f..1ae2f9ac 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/config/PersistenceConfig.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/config/PersistenceConfig.scala @@ -5,10 +5,8 @@ import com.typesafe.config.Config import com.karasiq.common.configs.ConfigImplicits import com.karasiq.shadowcloud.providers.SessionProvider -final case class PersistenceConfig(rootConfig: Config, - sessionProvider: Class[SessionProvider], - journalPlugin: String, - snapshotPlugin: String) extends WrappedConfig +final case class PersistenceConfig(rootConfig: Config, sessionProvider: Class[SessionProvider], journalPlugin: String, snapshotPlugin: String) + extends WrappedConfig object PersistenceConfig extends WrappedConfigFactory[PersistenceConfig] with ConfigImplicits { def apply(config: Config) = { diff --git a/core/src/main/scala/com/karasiq/shadowcloud/config/ProvidersConfig.scala b/core/src/main/scala/com/karasiq/shadowcloud/config/ProvidersConfig.scala index 9cdd16d1..61679323 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/config/ProvidersConfig.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/config/ProvidersConfig.scala @@ -8,9 +8,8 @@ import scala.collection.JavaConverters._ private[shadowcloud] case class ProvidersConfig[T](rootConfig: Config, classes: Seq[(String, Class[T])]) extends WrappedConfig { def instances(implicit inst: ProviderInstantiator): Seq[(String, T)] = { - classes.map { - case (name, pClass) ⇒ - name → inst.getInstance(pClass) + classes.map { case (name, pClass) ⇒ + name → inst.getInstance(pClass) } } } @@ -25,10 +24,9 @@ private[shadowcloud] object ProvidersConfig extends WrappedConfigFactory[Provide } private[this] def readProviders[T](obj: ConfigObject): Seq[(String, Class[T])] = { - obj.asScala.toVector.map { - case (key, value) ⇒ - require(value.valueType() == ConfigValueType.STRING, s"Invalid provider name: $value") - key → Class.forName(value.unwrapped().asInstanceOf[String]).asInstanceOf[Class[T]] + obj.asScala.toVector.map { case (key, value) ⇒ + require(value.valueType() == ConfigValueType.STRING, s"Invalid provider name: $value") + key → Class.forName(value.unwrapped().asInstanceOf[String]).asInstanceOf[Class[T]] } } } diff --git a/core/src/main/scala/com/karasiq/shadowcloud/config/QueuesConfig.scala b/core/src/main/scala/com/karasiq/shadowcloud/config/QueuesConfig.scala index f515cb06..70d3d27b 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/config/QueuesConfig.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/config/QueuesConfig.scala @@ -6,12 +6,14 @@ import com.typesafe.config.Config import com.karasiq.common.configs.ConfigImplicits -case class QueuesConfig(rootConfig: Config, - regionRepair: Int, - regionDiffs: Int, - regionDiffsTime: FiniteDuration, - chunksIndex: Int, - chunksIndexTime: FiniteDuration) extends WrappedConfig +case class QueuesConfig( + rootConfig: Config, + regionRepair: Int, + regionDiffs: Int, + regionDiffsTime: FiniteDuration, + chunksIndex: Int, + chunksIndexTime: FiniteDuration +) extends WrappedConfig object QueuesConfig extends ConfigImplicits with WrappedConfigFactory[QueuesConfig] { def apply(config: Config): QueuesConfig = { diff --git a/core/src/main/scala/com/karasiq/shadowcloud/config/RegionConfig.scala b/core/src/main/scala/com/karasiq/shadowcloud/config/RegionConfig.scala index 7d3c1edc..f4bea629 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/config/RegionConfig.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/config/RegionConfig.scala @@ -10,19 +10,22 @@ import com.karasiq.shadowcloud.storage.utils.ChunkKeyMapper import com.karasiq.shadowcloud.utils.Utils @SerialVersionUID(0L) -final case class RegionConfig(rootConfig: Config, - chunkKey: Option[ChunkKeyMapper], - chunkSize: Option[Int], - storageSelector: Class[StorageSelector], - dataReplicationFactor: Int, - indexReplicationFactor: Int, - garbageCollector: GCConfig) extends WrappedConfig +final case class RegionConfig( + rootConfig: Config, + chunkKey: Option[ChunkKeyMapper], + chunkSize: Option[Int], + storageSelector: Class[StorageSelector], + dataReplicationFactor: Int, + indexReplicationFactor: Int, + garbageCollector: GCConfig +) extends WrappedConfig object RegionConfig extends WrappedConfigFactory[RegionConfig] with ConfigImplicits { val empty = apply(Utils.emptyConfig) def forId(regionId: RegionId, rootConfig: Config): RegionConfig = { - val cfgObject = rootConfig.getConfigOrRef(s"regions.$regionId") + val cfgObject = rootConfig + .getConfigOrRef(s"regions.$regionId") .withFallback(rootConfig.getConfig("default-region")) apply(cfgObject) } diff --git a/core/src/main/scala/com/karasiq/shadowcloud/config/SerializationConfig.scala b/core/src/main/scala/com/karasiq/shadowcloud/config/SerializationConfig.scala index b53888b5..1a03ee41 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/config/SerializationConfig.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/config/SerializationConfig.scala @@ -4,18 +4,21 @@ import com.karasiq.common.configs.ConfigImplicits import com.karasiq.shadowcloud.compression.StreamCompression import com.karasiq.shadowcloud.config.SerializationConfig.Config -case class SerializationConfig(rootConfig: Config, - frameLimit: Int, - compression: StreamCompression.CompressionType.Value, - indexFormat: String, - keyFormat: String) extends WrappedConfig +case class SerializationConfig( + rootConfig: Config, + frameLimit: Int, + compression: StreamCompression.CompressionType.Value, + indexFormat: String, + keyFormat: String +) extends WrappedConfig object SerializationConfig extends WrappedConfigFactory[SerializationConfig] with ConfigImplicits { def apply(config: Config): SerializationConfig = { SerializationConfig( config, config.getBytesInt("frame-limit"), - config.optional(_.getString("compression")) + config + .optional(_.getString("compression")) .map(StreamCompression.CompressionType.withName) .getOrElse(StreamCompression.CompressionType.none), config.withDefault("default", _.getString("index-format")), diff --git a/core/src/main/scala/com/karasiq/shadowcloud/config/SigningConfig.scala b/core/src/main/scala/com/karasiq/shadowcloud/config/SigningConfig.scala index be6d2e23..40ea2e6f 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/config/SigningConfig.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/config/SigningConfig.scala @@ -1,7 +1,5 @@ package com.karasiq.shadowcloud.config - - import com.karasiq.common.configs.ConfigImplicits import com.karasiq.shadowcloud.model.crypto.{HashingMethod, SignMethod} import com.typesafe.config.{Config, ConfigException} @@ -19,9 +17,10 @@ object SigningConfig extends WrappedConfigFactory[SigningConfig] with ConfigImpl private[this] def getSignMethod(config: Config, path: String): SignMethod = { try { CryptoProps.signing(config.getConfigOrRef(path)) - } catch { case _: ConfigException.Missing ⇒ - val alg = config.getString(path) - SignMethod(alg, HashingMethod.default) + } catch { + case _: ConfigException.Missing ⇒ + val alg = config.getString(path) + SignMethod(alg, HashingMethod.default) } } } diff --git a/core/src/main/scala/com/karasiq/shadowcloud/config/StorageIndexConfig.scala b/core/src/main/scala/com/karasiq/shadowcloud/config/StorageIndexConfig.scala index d6736288..015c151c 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/config/StorageIndexConfig.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/config/StorageIndexConfig.scala @@ -7,13 +7,15 @@ import com.typesafe.config.Config import com.karasiq.common.configs.ConfigImplicits @SerialVersionUID(0L) -final case class StorageIndexConfig(rootConfig: Config, - syncInterval: FiniteDuration, - snapshotThreshold: Int, - snapshotClearHistory: Boolean, - compactThreshold: Int, - compactDeleteOld: Boolean, - keepDeleted: Boolean) +final case class StorageIndexConfig( + rootConfig: Config, + syncInterval: FiniteDuration, + snapshotThreshold: Int, + snapshotClearHistory: Boolean, + compactThreshold: Int, + compactDeleteOld: Boolean, + keepDeleted: Boolean +) object StorageIndexConfig extends ConfigImplicits { def apply(config: Config): StorageIndexConfig = { diff --git a/core/src/main/scala/com/karasiq/shadowcloud/config/StoragesConfig.scala b/core/src/main/scala/com/karasiq/shadowcloud/config/StoragesConfig.scala index b6083a2e..57e7ef7f 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/config/StoragesConfig.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/config/StoragesConfig.scala @@ -1,14 +1,10 @@ package com.karasiq.shadowcloud.config - - - import com.karasiq.common.configs.ConfigImplicits import com.karasiq.shadowcloud.providers.StorageProvider import com.typesafe.config.Config -private[shadowcloud] case class StoragesConfig(rootConfig: Config, - providers: ProvidersConfig[StorageProvider]) extends WrappedConfig +private[shadowcloud] case class StoragesConfig(rootConfig: Config, providers: ProvidersConfig[StorageProvider]) extends WrappedConfig private[shadowcloud] object StoragesConfig extends WrappedConfigFactory[StoragesConfig] with ConfigImplicits { def apply(config: Config): StoragesConfig = { diff --git a/core/src/main/scala/com/karasiq/shadowcloud/config/TimeoutsConfig.scala b/core/src/main/scala/com/karasiq/shadowcloud/config/TimeoutsConfig.scala index 68de4c98..6920fcc2 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/config/TimeoutsConfig.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/config/TimeoutsConfig.scala @@ -5,19 +5,21 @@ import scala.concurrent.duration.FiniteDuration import com.karasiq.common.configs.ConfigImplicits import com.karasiq.shadowcloud.config.QueuesConfig.Config -case class TimeoutsConfig(rootConfig: Config, - query: FiniteDuration, - chunkWrite: FiniteDuration, - chunkRead: FiniteDuration, - chunksList: FiniteDuration, - chunksDelete: FiniteDuration, - regionChunkWrite: FiniteDuration, - regionChunkRead: FiniteDuration, - indexWrite: FiniteDuration, - indexRead: FiniteDuration, - indexList: FiniteDuration, - synchronize: FiniteDuration, - collectGarbage: FiniteDuration) extends WrappedConfig +case class TimeoutsConfig( + rootConfig: Config, + query: FiniteDuration, + chunkWrite: FiniteDuration, + chunkRead: FiniteDuration, + chunksList: FiniteDuration, + chunksDelete: FiniteDuration, + regionChunkWrite: FiniteDuration, + regionChunkRead: FiniteDuration, + indexWrite: FiniteDuration, + indexRead: FiniteDuration, + indexList: FiniteDuration, + synchronize: FiniteDuration, + collectGarbage: FiniteDuration +) extends WrappedConfig object TimeoutsConfig extends ConfigImplicits with WrappedConfigFactory[TimeoutsConfig] { def apply(config: Config): TimeoutsConfig = { diff --git a/core/src/main/scala/com/karasiq/shadowcloud/crypto/NoOpCryptoProvider.scala b/core/src/main/scala/com/karasiq/shadowcloud/crypto/NoOpCryptoProvider.scala index ea7106bb..ed110ea4 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/crypto/NoOpCryptoProvider.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/crypto/NoOpCryptoProvider.scala @@ -1,7 +1,5 @@ package com.karasiq.shadowcloud.crypto - - import com.karasiq.common.configs.ConfigImplicits import com.karasiq.shadowcloud.crypto.internal.{NoOpEncryptionModule, NoOpHashingModule, NoOpSignModule} import com.karasiq.shadowcloud.model.crypto.CryptoMethod diff --git a/core/src/main/scala/com/karasiq/shadowcloud/crypto/index/IndexEncryption.scala b/core/src/main/scala/com/karasiq/shadowcloud/crypto/index/IndexEncryption.scala index 809cdb31..cf2dad12 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/crypto/index/IndexEncryption.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/crypto/index/IndexEncryption.scala @@ -26,13 +26,13 @@ private[shadowcloud] trait IndexEncryption { private[shadowcloud] object IndexEncryption { private[crypto] type DataId = UUID private[crypto] type NonceT = ByteString - type PlaintextT = ByteString - type CiphertextT = EncryptedIndexData + type PlaintextT = ByteString + type CiphertextT = EncryptedIndexData def apply(cryptoModules: CryptoModuleRegistry, serialization: IndexSerialization): IndexEncryption = { new DefaultIndexEncryption(cryptoModules, serialization) } - + def getKeyHash(dataId: DataId, keyId: KeyId): Int = { MurmurHash3.arrayHash((UUIDEncoding.toBytes(keyId) ++ UUIDEncoding.toBytes(dataId)).toArray) } @@ -57,16 +57,16 @@ private[shadowcloud] object IndexEncryption { } } -private[shadowcloud] final class DefaultIndexEncryption(cryptoModules: CryptoModuleRegistry, - serialization: IndexSerialization) extends IndexEncryption { +private[shadowcloud] final class DefaultIndexEncryption(cryptoModules: CryptoModuleRegistry, serialization: IndexSerialization) + extends IndexEncryption { private[this] lazy val secureRandom = new SecureRandom() def encrypt(plaintext: ByteString, dataEncMethod: EncryptionMethod, keys: KeyChain): EncryptedIndexData = { def createEncryptedData(plaintext: ByteString, method: EncryptionMethod): (EncryptedIndexData, EncryptionParameters) = { - val dataId = UUID.randomUUID() - val dataEncModule = cryptoModules.encryptionModule(method) + val dataId = UUID.randomUUID() + val dataEncModule = cryptoModules.encryptionModule(method) val dataEncParameters = dataEncModule.createParameters() - val ciphertext = dataEncModule.encrypt(plaintext, dataEncParameters) + val ciphertext = dataEncModule.encrypt(plaintext, dataEncParameters) (EncryptedIndexData(id = dataId, data = ciphertext), dataEncParameters) } @@ -82,12 +82,12 @@ private[shadowcloud] final class DefaultIndexEncryption(cryptoModules: CryptoMod def createHeader(encData: EncryptedIndexData, dataEncParameters: EncryptionParameters, staticKeys: KeySet): EncryptedIndexData.Header = { val keyEncModule = cryptoModules.encryptionModule(staticKeys.encryption.method) - val signatures = IndexSignatures(cryptoModules, staticKeys.signing.method) + val signatures = IndexSignatures(cryptoModules, staticKeys.signing.method) - val dataNonce = generateNonce(staticKeys.encryption) + val dataNonce = generateNonce(staticKeys.encryption) val keyEncParameters = IndexEncryption.updateNonce(staticKeys.encryption, dataNonce) - val serializedKey = serialization.wrapKey(dataEncParameters) + val serializedKey = serialization.wrapKey(dataEncParameters) val headerCiphertext = keyEncModule.encrypt(ByteString.fromArrayUnsafe(serializedKey.toByteArray), keyEncParameters) val header = EncryptedIndexData.Header( keyHash = IndexEncryption.getKeyHash(encData.id, staticKeys.id), @@ -99,7 +99,7 @@ private[shadowcloud] final class DefaultIndexEncryption(cryptoModules: CryptoMod } val (encData, dataEncParameters) = createEncryptedData(plaintext, dataEncMethod) - val headers = keys.encKeys.map(keySet ⇒ createHeader(encData, dataEncParameters, keySet)) + val headers = keys.encKeys.map(keySet ⇒ createHeader(encData, dataEncParameters, keySet)) encData.copy(headers = headers.toVector) } @@ -118,11 +118,11 @@ private[shadowcloud] final class DefaultIndexEncryption(cryptoModules: CryptoMod val (keySet, header) = matchingKeys.next() val keyEncParameters = IndexEncryption.updateNonce(keySet.encryption, header.nonce) - val keyEncModule = cryptoModules.encryptionModule(keyEncParameters.method) + val keyEncModule = cryptoModules.encryptionModule(keyEncParameters.method) - val decryptedKey = SerializedKeyData.parseFrom(keyEncModule.decrypt(header.data, keyEncParameters).toArrayUnsafe) + val decryptedKey = SerializedKeyData.parseFrom(keyEncModule.decrypt(header.data, keyEncParameters).toArrayUnsafe) val dataEncParameters = serialization.unwrapKey(decryptedKey) - val dataEncModule = cryptoModules.encryptionModule(dataEncParameters.method) + val dataEncModule = cryptoModules.encryptionModule(dataEncParameters.method) dataEncModule.decrypt(data.data, dataEncParameters) } diff --git a/core/src/main/scala/com/karasiq/shadowcloud/crypto/index/IndexSignatures.scala b/core/src/main/scala/com/karasiq/shadowcloud/crypto/index/IndexSignatures.scala index 1e7c1e23..5a5fd4a4 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/crypto/index/IndexSignatures.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/crypto/index/IndexSignatures.scala @@ -16,7 +16,7 @@ private[shadowcloud] trait IndexSignatures { private[shadowcloud] object IndexSignatures { type PayloadT = EncryptedIndexData - type HeaderT = EncryptedIndexData.Header + type HeaderT = EncryptedIndexData.Header def createPayload(data: EncryptedIndexData, header: EncryptedIndexData.Header): ByteString = { UUIDEncoding.toBytes(data.id) ++ header.nonce ++ header.data ++ data.data @@ -33,7 +33,7 @@ private[shadowcloud] object IndexSignatures { private[shadowcloud] final class DefaultIndexSignatures(signModule: SignModule) extends IndexSignatures { def sign(data: PayloadT, header: HeaderT, signParameters: SignParameters): HeaderT = { - val payload = IndexSignatures.createPayload(data, header) + val payload = IndexSignatures.createPayload(data, header) val signature = signModule.sign(payload, signParameters) header.copy(signature = signature) } diff --git a/core/src/main/scala/com/karasiq/shadowcloud/crypto/internal/NoOpEncryptionModule.scala b/core/src/main/scala/com/karasiq/shadowcloud/crypto/internal/NoOpEncryptionModule.scala index b626607f..2a3acce2 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/crypto/internal/NoOpEncryptionModule.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/crypto/internal/NoOpEncryptionModule.scala @@ -1,7 +1,5 @@ package com.karasiq.shadowcloud.crypto.internal - - import akka.util.ByteString import com.karasiq.shadowcloud.crypto._ import com.karasiq.shadowcloud.model.crypto.{EncryptionMethod, EncryptionParameters} @@ -9,8 +7,8 @@ import com.karasiq.shadowcloud.model.crypto.{EncryptionMethod, EncryptionParamet private[crypto] final class NoOpEncryptionModule extends StreamEncryptionModule { def method: EncryptionMethod = EncryptionMethod.none - def createParameters(): EncryptionParameters = EncryptionParameters.empty - def updateParameters(parameters: EncryptionParameters): EncryptionParameters = parameters + def createParameters(): EncryptionParameters = EncryptionParameters.empty + def updateParameters(parameters: EncryptionParameters): EncryptionParameters = parameters override def encrypt(data: ByteString, parameters: EncryptionParameters): ByteString = data override def decrypt(data: ByteString, parameters: EncryptionParameters): ByteString = data @@ -18,9 +16,9 @@ private[crypto] final class NoOpEncryptionModule extends StreamEncryptionModule def createStreamer(): EncryptionModuleStreamer = NoOpEncryptionStreamer private[this] object NoOpEncryptionStreamer extends EncryptionModuleStreamer { - def module: EncryptionModule = NoOpEncryptionModule.this + def module: EncryptionModule = NoOpEncryptionModule.this def init(encrypt: Boolean, parameters: EncryptionParameters): Unit = () - def process(data: ByteString): ByteString = data - def finish(): ByteString = ByteString.empty + def process(data: ByteString): ByteString = data + def finish(): ByteString = ByteString.empty } } diff --git a/core/src/main/scala/com/karasiq/shadowcloud/crypto/internal/NoOpHashingModule.scala b/core/src/main/scala/com/karasiq/shadowcloud/crypto/internal/NoOpHashingModule.scala index 2d99fef4..2a6a6d26 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/crypto/internal/NoOpHashingModule.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/crypto/internal/NoOpHashingModule.scala @@ -1,20 +1,18 @@ package com.karasiq.shadowcloud.crypto.internal - - import akka.util.ByteString import com.karasiq.shadowcloud.crypto.{HashingModule, HashingModuleStreamer, StreamHashingModule} import com.karasiq.shadowcloud.model.crypto.HashingMethod private[crypto] final class NoOpHashingModule extends StreamHashingModule { - def method: HashingMethod = HashingMethod.none - def createHash(data: ByteString) = ByteString.empty + def method: HashingMethod = HashingMethod.none + def createHash(data: ByteString) = ByteString.empty def createStreamer(): HashingModuleStreamer = NoOpHashingStreamer private[this] object NoOpHashingStreamer extends HashingModuleStreamer { - def module: HashingModule = NoOpHashingModule.this + def module: HashingModule = NoOpHashingModule.this def update(data: ByteString): Unit = () - def reset(): Unit = () - def finish(): ByteString = ByteString.empty + def reset(): Unit = () + def finish(): ByteString = ByteString.empty } } diff --git a/core/src/main/scala/com/karasiq/shadowcloud/crypto/internal/NoOpSignModule.scala b/core/src/main/scala/com/karasiq/shadowcloud/crypto/internal/NoOpSignModule.scala index b713eb75..3198e933 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/crypto/internal/NoOpSignModule.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/crypto/internal/NoOpSignModule.scala @@ -1,23 +1,21 @@ package com.karasiq.shadowcloud.crypto.internal - - import akka.util.ByteString import com.karasiq.shadowcloud.crypto._ import com.karasiq.shadowcloud.model.crypto.{SignMethod, SignParameters} private[crypto] final class NoOpSignModule extends StreamSignModule { - def method: SignMethod = SignMethod.none - def createParameters(): SignParameters = SignParameters.empty - def sign(data: ByteString, parameters: SignParameters) = ByteString.empty + def method: SignMethod = SignMethod.none + def createParameters(): SignParameters = SignParameters.empty + def sign(data: ByteString, parameters: SignParameters) = ByteString.empty def verify(data: ByteString, signature: ByteString, parameters: SignParameters) = signature.isEmpty - def createStreamer(): SignModuleStreamer = NoOpSignStreamer + def createStreamer(): SignModuleStreamer = NoOpSignStreamer private[this] object NoOpSignStreamer extends SignModuleStreamer { - def module: SignModule = NoOpSignModule.this + def module: SignModule = NoOpSignModule.this def init(sign: Boolean, parameters: SignParameters): Unit = () - def update(data: ByteString): Unit = () - def finishVerify(signature: ByteString): Boolean = signature.isEmpty - def finishSign(): ByteString = ByteString.empty + def update(data: ByteString): Unit = () + def finishVerify(signature: ByteString): Boolean = signature.isEmpty + def finishSign(): ByteString = ByteString.empty } } diff --git a/core/src/main/scala/com/karasiq/shadowcloud/internal/StdLifecycleHook.scala b/core/src/main/scala/com/karasiq/shadowcloud/internal/StdLifecycleHook.scala index c44ebd48..5141899e 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/internal/StdLifecycleHook.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/internal/StdLifecycleHook.scala @@ -15,16 +15,16 @@ private class StdLifecycleHook(sc: ShadowCloudExtension) extends LifecycleHook { import sc.implicits.executionContext val syncFuture = if (sc.config.misc.syncOnExit) { - for { - regions <- sc.ops.supervisor.getSnapshot() - _ <- Future.sequence( - regions.regions.filter(_._2.actorState.isInstanceOf[ActorState.Active]).keys.map(regionId => sc.ops.region.synchronize(regionId)) - ) - } yield Done + for { + regions ← sc.ops.supervisor.getSnapshot() + _ ← Future.sequence( + regions.regions.filter(_._2.actorState.isInstanceOf[ActorState.Active]).keys.map(regionId ⇒ sc.ops.region.synchronize(regionId)) + ) + } yield Done } else Future.successful(Done) val future = for { - _ <- syncFuture + _ ← syncFuture _ = sc.implicits.actorSystem.stop(sc.actors.regionSupervisor) } yield NotUsed diff --git a/core/src/main/scala/com/karasiq/shadowcloud/metadata/MetadataUtils.scala b/core/src/main/scala/com/karasiq/shadowcloud/metadata/MetadataUtils.scala index c424fc6a..84414410 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/metadata/MetadataUtils.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/metadata/MetadataUtils.scala @@ -25,7 +25,8 @@ private[shadowcloud] object MetadataUtils { } def expiredFileIds(index: FolderIndex): Set[FileId] = { - val metadataFolders = index.get(MetadataFolder) + val metadataFolders = index + .get(MetadataFolder) .map(_.folders) .getOrElse(Set.empty) diff --git a/core/src/main/scala/com/karasiq/shadowcloud/ops/region/RegionOps.scala b/core/src/main/scala/com/karasiq/shadowcloud/ops/region/RegionOps.scala index 561b63f7..9e7c7092 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/ops/region/RegionOps.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/ops/region/RegionOps.scala @@ -27,20 +27,18 @@ import com.karasiq.shadowcloud.streams.chunk.ChunkProcessingStreams import scala.concurrent.{ExecutionContext, Future} object RegionOps { - def apply(regionSupervisor: ActorRef, - timeouts: TimeoutsConfig, - chunkCache: ChunkCache, - chunkProcessing: ChunkProcessingStreams) - (implicit ec: ExecutionContext, mat: Materializer): RegionOps = { + def apply(regionSupervisor: ActorRef, timeouts: TimeoutsConfig, chunkCache: ChunkCache, chunkProcessing: ChunkProcessingStreams)(implicit + ec: ExecutionContext, + mat: Materializer + ): RegionOps = { new RegionOps(regionSupervisor, timeouts, chunkCache, chunkProcessing) } } -final class RegionOps(regionSupervisor: ActorRef, - timeouts: TimeoutsConfig, - chunkCache: ChunkCache, - chunkProcessing: ChunkProcessingStreams) - (implicit ec: ExecutionContext, mat: Materializer) { +final class RegionOps(regionSupervisor: ActorRef, timeouts: TimeoutsConfig, chunkCache: ChunkCache, chunkProcessing: ChunkProcessingStreams)(implicit + ec: ExecutionContext, + mat: Materializer +) { // ----------------------------------------------------------------------- // Index @@ -146,11 +144,15 @@ final class RegionOps(regionSupervisor: ActorRef, // TODO: Fix crypto parallelism not applied def readChunk(regionId: RegionId, chunk: Chunk): Future[Chunk] = { - chunkCache.readCached(chunk, { () ⇒ - Source.future(readChunkEncrypted(regionId, chunk)) - .via(chunkProcessing.afterRead) - .runWith(Sink.head) - }) + chunkCache.readCached( + chunk, + { () ⇒ + Source + .future(readChunkEncrypted(regionId, chunk)) + .via(chunkProcessing.afterRead) + .runWith(Sink.head) + } + ) } def rewriteChunk(regionId: RegionId, chunk: Chunk, newAffinity: Option[ChunkWriteAffinity]): Future[Chunk] = { @@ -178,8 +180,9 @@ final class RegionOps(regionSupervisor: ActorRef, // ----------------------------------------------------------------------- // Utils // ----------------------------------------------------------------------- - private[this] def askRegion[V](regionId: RegionId, status: MessageStatus[_, V], message: Any) - (implicit timeout: Timeout = timeouts.query): Future[V] = { + private[this] def askRegion[V](regionId: RegionId, status: MessageStatus[_, V], message: Any)(implicit + timeout: Timeout = timeouts.query + ): Future[V] = { status.unwrapFuture(regionSupervisor ? RegionEnvelope(regionId, message)) } } diff --git a/core/src/main/scala/com/karasiq/shadowcloud/ops/storage/StorageOps.scala b/core/src/main/scala/com/karasiq/shadowcloud/ops/storage/StorageOps.scala index fe468522..04d62597 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/ops/storage/StorageOps.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/ops/storage/StorageOps.scala @@ -76,14 +76,15 @@ final class StorageOps(regionSupervisor: ActorRef, timeouts: TimeoutsConfig)(imp askStorage(storageId, StorageDispatcher.GetHealth, StorageDispatcher.GetHealth(checkNow)) } - private[this] def askStorage[V](storageId: StorageId, status: MessageStatus[_, V], message: Any) - (implicit timeout: Timeout = timeouts.query): Future[V] = { + private[this] def askStorage[V](storageId: StorageId, status: MessageStatus[_, V], message: Any)(implicit + timeout: Timeout = timeouts.query + ): Future[V] = { status.unwrapFuture(regionSupervisor ? StorageEnvelope(storageId, message)) } - private[this] def askStorageIndex[V](storageId: StorageId, regionId: RegionId, - status: MessageStatus[_, V], message: RegionIndex.Message) - (implicit timeout: Timeout = timeouts.query): Future[V] = { + private[this] def askStorageIndex[V](storageId: StorageId, regionId: RegionId, status: MessageStatus[_, V], message: RegionIndex.Message)(implicit + timeout: Timeout = timeouts.query + ): Future[V] = { askStorage(storageId, status, StorageIndex.Envelope(regionId, message))(timeout) } } diff --git a/core/src/main/scala/com/karasiq/shadowcloud/ops/supervisor/RegionSupervisorOps.scala b/core/src/main/scala/com/karasiq/shadowcloud/ops/supervisor/RegionSupervisorOps.scala index 155b6364..b73bb659 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/ops/supervisor/RegionSupervisorOps.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/ops/supervisor/RegionSupervisorOps.scala @@ -61,7 +61,7 @@ final class RegionSupervisorOps(regionSupervisor: ActorRef, timeouts: TimeoutsCo GetSnapshot.unwrapFuture(regionSupervisor.ask(GetSnapshot)(timeouts.query)) } - def getRegionConfig(regionId: RegionId): Future[RegionConfig] ={ + def getRegionConfig(regionId: RegionId): Future[RegionConfig] = { getSnapshot().map(_.regions(regionId).regionConfig) } } diff --git a/core/src/main/scala/com/karasiq/shadowcloud/providers/CryptoModuleRegistry.scala b/core/src/main/scala/com/karasiq/shadowcloud/providers/CryptoModuleRegistry.scala index 546e22ff..e0201a86 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/providers/CryptoModuleRegistry.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/providers/CryptoModuleRegistry.scala @@ -59,18 +59,20 @@ private[shadowcloud] object CryptoModuleRegistry { } } -private[providers] final class CryptoModuleRegistryImpl(providers: ProvidersConfig[CryptoProvider]) - (implicit inst: ProviderInstantiator) extends CryptoModuleRegistry { +private[providers] final class CryptoModuleRegistryImpl(providers: ProvidersConfig[CryptoProvider])(implicit inst: ProviderInstantiator) + extends CryptoModuleRegistry { // ----------------------------------------------------------------------- // Context // ----------------------------------------------------------------------- - private[this] case class Context(hashing: PartialFunction[HashingMethod, HashingModule] = PartialFunction.empty, - encryption: PartialFunction[EncryptionMethod, EncryptionModule] = PartialFunction.empty, - signing: PartialFunction[SignMethod, SignModule] = PartialFunction.empty) + private[this] case class Context( + hashing: PartialFunction[HashingMethod, HashingModule] = PartialFunction.empty, + encryption: PartialFunction[EncryptionMethod, EncryptionModule] = PartialFunction.empty, + signing: PartialFunction[SignMethod, SignModule] = PartialFunction.empty + ) private[this] val providerInstances = providers.instances - private[this] val providerMap = providerInstances.toMap + private[this] val providerMap = providerInstances.toMap private[this] val context = providerInstances.foldLeft(Context()) { case (ctx, (_, pr)) ⇒ ctx.copy( pr.hashing.orElse(ctx.hashing), diff --git a/core/src/main/scala/com/karasiq/shadowcloud/providers/MetadataModuleRegistry.scala b/core/src/main/scala/com/karasiq/shadowcloud/providers/MetadataModuleRegistry.scala index b702322b..373c1f98 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/providers/MetadataModuleRegistry.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/providers/MetadataModuleRegistry.scala @@ -20,8 +20,8 @@ private[shadowcloud] object MetadataModuleRegistry { } } -private[shadowcloud] final class MetadataModuleRegistryImpl(providers: ProvidersConfig[MetadataProvider]) - (implicit inst: ProviderInstantiator) extends MetadataModuleRegistry { +private[shadowcloud] final class MetadataModuleRegistryImpl(providers: ProvidersConfig[MetadataProvider])(implicit inst: ProviderInstantiator) + extends MetadataModuleRegistry { private[this] val (plugins, detectors, parsers) = { val (detectors, parsers) = providers.instances @@ -50,7 +50,7 @@ private[shadowcloud] final class MetadataModuleRegistryImpl(providers: Providers val graph = GraphDSL.create() { implicit builder ⇒ import GraphDSL.Implicits._ val broadcast = builder.add(Broadcast[ByteString](availableParsers.length)) - val merge = builder.add(Merge[Metadata](availableParsers.length)) + val merge = builder.add(Merge[Metadata](availableParsers.length)) availableParsers.foreach { parser ⇒ val parse = builder.add(parser.parseMetadata(name, mime).withAttributes(parserAttributes)) broadcast ~> parse ~> merge diff --git a/core/src/main/scala/com/karasiq/shadowcloud/providers/SCModules.scala b/core/src/main/scala/com/karasiq/shadowcloud/providers/SCModules.scala index b08d326c..3f500c6f 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/providers/SCModules.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/providers/SCModules.scala @@ -1,7 +1,5 @@ package com.karasiq.shadowcloud.providers - - import com.karasiq.shadowcloud.config.{ProvidersConfig, SCConfig} import com.karasiq.shadowcloud.metadata.MetadataProvider import com.karasiq.shadowcloud.utils.ProviderInstantiator @@ -22,16 +20,19 @@ private[shadowcloud] object SCModules { } } -private[shadowcloud] final class SCModulesImpl(_storages: ProvidersConfig[StorageProvider], - _crypto: ProvidersConfig[CryptoProvider], - _metadata: ProvidersConfig[MetadataProvider]) - (implicit inst: ProviderInstantiator) extends SCModules { +private[shadowcloud] final class SCModulesImpl( + _storages: ProvidersConfig[StorageProvider], + _crypto: ProvidersConfig[CryptoProvider], + _metadata: ProvidersConfig[MetadataProvider] +)(implicit inst: ProviderInstantiator) + extends SCModules { - val storage = StorageModuleRegistry(_storages) - val crypto = CryptoModuleRegistry(_crypto) + val storage = StorageModuleRegistry(_storages) + val crypto = CryptoModuleRegistry(_crypto) val metadata = MetadataModuleRegistry(_metadata) override def toString: String = { - s"SCModules(storages = [${storage.storageTypes.mkString(", ")}, hashes = [${crypto.hashingAlgorithms.mkString(", ")}], encryption = [${crypto.encryptionAlgorithms.mkString(", ")}], metadata = [${metadata.metadataPlugins.mkString(", ")}])" + s"SCModules(storages = [${storage.storageTypes.mkString(", ")}, hashes = [${crypto.hashingAlgorithms.mkString(", ")}], encryption = [${crypto.encryptionAlgorithms + .mkString(", ")}], metadata = [${metadata.metadataPlugins.mkString(", ")}])" } } diff --git a/core/src/main/scala/com/karasiq/shadowcloud/serialization/IndexSerialization.scala b/core/src/main/scala/com/karasiq/shadowcloud/serialization/IndexSerialization.scala index a8afa0f8..2f144407 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/serialization/IndexSerialization.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/serialization/IndexSerialization.scala @@ -24,7 +24,7 @@ private[shadowcloud] object IndexSerialization { } } -private[shadowcloud] final class DefaultIndexSerialization(implicit sc: ShadowCloudExtension) extends IndexSerialization { +private[shadowcloud] final class DefaultIndexSerialization(implicit sc: ShadowCloudExtension) extends IndexSerialization { def wrapIndexFrame(data: IndexData): SerializedIndexData = { val format = sc.config.serialization.indexFormat val bytes = format match { diff --git a/core/src/main/scala/com/karasiq/shadowcloud/storage/LocalStorageProvider.scala b/core/src/main/scala/com/karasiq/shadowcloud/storage/LocalStorageProvider.scala index 447c8e48..da016a05 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/storage/LocalStorageProvider.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/storage/LocalStorageProvider.scala @@ -1,7 +1,5 @@ package com.karasiq.shadowcloud.storage - - import com.karasiq.shadowcloud.config.ConfigProps import com.karasiq.shadowcloud.providers.StorageProvider import com.karasiq.shadowcloud.storage.files.FileStoragePlugin diff --git a/core/src/main/scala/com/karasiq/shadowcloud/storage/Repositories.scala b/core/src/main/scala/com/karasiq/shadowcloud/storage/Repositories.scala index 2e378167..cdcc990a 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/storage/Repositories.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/storage/Repositories.scala @@ -8,11 +8,10 @@ import com.karasiq.shadowcloud.storage.files.FileRepository import com.karasiq.shadowcloud.storage.inmem.ConcurrentMapRepository import com.karasiq.shadowcloud.storage.repository.{PathTreeRepository, Repository} -import scala.collection.concurrent.{TrieMap, Map => CMap} +import scala.collection.concurrent.{TrieMap, Map ⇒ CMap} import scala.concurrent.ExecutionContext -/** - * Standard repositories +/** Standard repositories */ private[shadowcloud] object Repositories { def fromConcurrentMap[T](map: CMap[T, ByteString]): Repository[T] = { diff --git a/core/src/main/scala/com/karasiq/shadowcloud/storage/StorageHealthProviders.scala b/core/src/main/scala/com/karasiq/shadowcloud/storage/StorageHealthProviders.scala index 35a6b315..790547f1 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/storage/StorageHealthProviders.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/storage/StorageHealthProviders.scala @@ -11,8 +11,7 @@ import scala.collection.mutable import scala.concurrent.ExecutionContext private[shadowcloud] object StorageHealthProviders { - def fromDirectory(directory: Path, quota: StorageProps.Quota = StorageProps.Quota.empty) - (implicit ec: ExecutionContext): StorageHealthProvider = { + def fromDirectory(directory: Path, quota: StorageProps.Quota = StorageProps.Quota.empty)(implicit ec: ExecutionContext): StorageHealthProvider = { if (quota.isEmpty) { // Quickly checks drive free space @@ -28,6 +27,6 @@ private[shadowcloud] object StorageHealthProviders { } def fromMaps(maps: mutable.Map[_, ByteString]*): StorageHealthProvider = { - fromMaps(StorageProps.Quota.empty, maps:_*) + fromMaps(StorageProps.Quota.empty, maps: _*) } } diff --git a/core/src/main/scala/com/karasiq/shadowcloud/storage/files/FileRepository.scala b/core/src/main/scala/com/karasiq/shadowcloud/storage/files/FileRepository.scala index 5e434011..998acb5e 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/storage/files/FileRepository.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/storage/files/FileRepository.scala @@ -1,7 +1,7 @@ package com.karasiq.shadowcloud.storage.files import java.io.FileNotFoundException -import java.nio.file.{Files, StandardOpenOption, Path => FSPath} +import java.nio.file.{Files, StandardOpenOption, Path ⇒ FSPath} import akka.NotUsed import akka.stream.scaladsl.{FileIO, Flow, Keep, Sink, Source} @@ -22,7 +22,8 @@ private[storage] object FileRepository { private def toSCPath(path: FSPath): Path = { val nodes = path - .iterator().asScala + .iterator() + .asScala .map(_.getFileName.toString) .toVector @@ -30,8 +31,7 @@ private[storage] object FileRepository { } } -/** - * Uses local filesystem to store data +/** Uses local filesystem to store data * @param rootFolder Root directory */ private[storage] final class FileRepository(rootFolder: FSPath)(implicit ec: ExecutionContext, mat: Materializer) extends PathTreeRepository { @@ -41,10 +41,11 @@ private[storage] final class FileRepository(rootFolder: FSPath)(implicit ec: Exe } def write(key: Path): Sink[Data, Result] = { - val path = toRealPath(key) + val path = toRealPath(key) val parentDir = path.getParent if (!Files.isDirectory(parentDir)) Files.createDirectories(parentDir) - FileIO.toPath(path, Set(StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) + FileIO + .toPath(path, Set(StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) .mapMaterializedValue(StorageUtils.wrapAkkaIOFuture(path.toString, _)) } @@ -62,7 +63,8 @@ private[storage] final class FileRepository(rootFolder: FSPath)(implicit ec: Exe def subKeys(fromPath: Path): Source[Path, Result] = { val subDirPath = toRealPath(fromPath) - FileSystemUtils.walkFileTree(subDirPath, includeDirs = false) + FileSystemUtils + .walkFileTree(subDirPath, includeDirs = false) .recoverWithRetries(1, { case _: FileNotFoundException ⇒ Source.empty }) // .log("file-repository-tree") .map(fsPath ⇒ toVirtualPath(fsPath).toRelative(fromPath)) @@ -80,7 +82,8 @@ private[storage] final class FileRepository(rootFolder: FSPath)(implicit ec: Exe FileRepository.toSCPath(rootFolder.relativize(fsPath)) } - private[this] val fileDeleteSink: Sink[FSPath, NotUsed] = Sink.foreach(Files.delete(_: FSPath)) + private[this] val fileDeleteSink: Sink[FSPath, NotUsed] = Sink + .foreach(Files.delete(_: FSPath)) .withAttributes(Attributes.name("fileDelete") and ActorAttributes.IODispatcher) .mapMaterializedValue(_ ⇒ NotUsed) } diff --git a/core/src/main/scala/com/karasiq/shadowcloud/storage/files/FileStorageEstimateHealthProvider.scala b/core/src/main/scala/com/karasiq/shadowcloud/storage/files/FileStorageEstimateHealthProvider.scala index c5bcceb0..f30de2a4 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/storage/files/FileStorageEstimateHealthProvider.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/storage/files/FileStorageEstimateHealthProvider.scala @@ -1,22 +1,21 @@ package com.karasiq.shadowcloud.storage.files -import java.nio.file.{Files, Path => FsPath} +import java.nio.file.{Files, Path ⇒ FsPath} import com.karasiq.shadowcloud.model.utils.StorageHealth import com.karasiq.shadowcloud.storage.StorageHealthProvider import scala.concurrent.{ExecutionContext, Future} -private[storage] final class FileStorageEstimateHealthProvider(directory: FsPath)(implicit ec: ExecutionContext) - extends StorageHealthProvider { +private[storage] final class FileStorageEstimateHealthProvider(directory: FsPath)(implicit ec: ExecutionContext) extends StorageHealthProvider { def health: Future[StorageHealth] = { Future { if (!Files.isDirectory(directory)) Files.createDirectories(directory) val fileStore = Files.getFileStore(directory) - val total = fileStore.getTotalSpace - val free = fileStore.getUsableSpace - val canWrite = if (fileStore.isReadOnly) 0 else free + val total = fileStore.getTotalSpace + val free = fileStore.getUsableSpace + val canWrite = if (fileStore.isReadOnly) 0 else free StorageHealth(canWrite, total, total - free) } } diff --git a/core/src/main/scala/com/karasiq/shadowcloud/storage/files/FileStorageTraverseHealthProvider.scala b/core/src/main/scala/com/karasiq/shadowcloud/storage/files/FileStorageTraverseHealthProvider.scala index f4642e50..15f1d207 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/storage/files/FileStorageTraverseHealthProvider.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/storage/files/FileStorageTraverseHealthProvider.scala @@ -1,6 +1,6 @@ package com.karasiq.shadowcloud.storage.files -import java.nio.file.{Files, Path => FsPath} +import java.nio.file.{Files, Path ⇒ FsPath} import com.karasiq.shadowcloud.model.utils.StorageHealth import com.karasiq.shadowcloud.storage.StorageHealthProvider @@ -14,10 +14,10 @@ private[storage] final class FileStorageTraverseHealthProvider(directory: FsPath Future { if (!Files.isDirectory(directory)) Files.createDirectories(directory) val fileStore = Files.getFileStore(directory) - val total = fileStore.getTotalSpace - val used = FileSystemUtils.getFolderSize(directory) - val free = math.max(total - used, 0L) - val canWrite = if (fileStore.isReadOnly) 0 else free + val total = fileStore.getTotalSpace + val used = FileSystemUtils.getFolderSize(directory) + val free = math.max(total - used, 0L) + val canWrite = if (fileStore.isReadOnly) 0 else free StorageHealth(canWrite, total, used) } } diff --git a/core/src/main/scala/com/karasiq/shadowcloud/storage/inmem/ConcurrentMapRepository.scala b/core/src/main/scala/com/karasiq/shadowcloud/storage/inmem/ConcurrentMapRepository.scala index 8049b1d6..a92193eb 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/storage/inmem/ConcurrentMapRepository.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/storage/inmem/ConcurrentMapRepository.scala @@ -5,10 +5,9 @@ import akka.util.ByteString import com.karasiq.shadowcloud.storage.repository.Repository import com.karasiq.shadowcloud.streams.utils.ByteStreams -import scala.collection.concurrent.{Map => CMap} +import scala.collection.concurrent.{Map ⇒ CMap} -/** - * Stores data in [[scala.collection.concurrent.TrieMap TrieMap]] +/** Stores data in [[scala.collection.concurrent.TrieMap TrieMap]] */ private[storage] final class ConcurrentMapRepository[Key](storage: CMap[Key, ByteString]) extends Repository[Key] { private[this] val underlying = new ConcurrentMapStreams[Key, ByteString](storage, _.length) diff --git a/core/src/main/scala/com/karasiq/shadowcloud/storage/inmem/ConcurrentMapStreams.scala b/core/src/main/scala/com/karasiq/shadowcloud/storage/inmem/ConcurrentMapStreams.scala index 44238110..f3ed8b74 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/storage/inmem/ConcurrentMapStreams.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/storage/inmem/ConcurrentMapStreams.scala @@ -7,12 +7,13 @@ import com.karasiq.shadowcloud.exceptions.StorageException import com.karasiq.shadowcloud.storage.StorageIOResult import com.karasiq.shadowcloud.storage.utils.StorageUtils -import scala.collection.concurrent.{Map => CMap} +import scala.collection.concurrent.{Map ⇒ CMap} import scala.concurrent.Future private[inmem] final class ConcurrentMapStreams[K, V](map: CMap[K, V], length: V ⇒ Int) { def keys: Source[K, Future[StorageIOResult]] = { - Source.fromIterator(() ⇒ map.keysIterator) + Source + .fromIterator(() ⇒ map.keysIterator) .alsoToMat(StorageUtils.countPassedElements().toMat(Sink.head)(Keep.right))(Keep.right) .named("concurrentMapKeys") } @@ -21,11 +22,13 @@ private[inmem] final class ConcurrentMapStreams[K, V](map: CMap[K, V], length: V val path = StorageUtils.toStoragePath(key) map.get(key) match { case Some(data) ⇒ - Source.single(data) + Source + .single(data) .mapMaterializedValue(_ ⇒ Future.successful(StorageIOResult.Success(path, length(data)))) case None ⇒ - Source.failed(StorageException.NotFound(path)) + Source + .failed(StorageException.NotFound(path)) .mapMaterializedValue(_ ⇒ Future.successful(StorageIOResult.Failure(path, StorageException.NotFound(path)))) } } @@ -47,12 +50,13 @@ private[inmem] final class ConcurrentMapStreams[K, V](map: CMap[K, V], length: V Flow[K] .map { key ⇒ val path = StorageUtils.toStoragePath(key) - map.remove(key) + map + .remove(key) .map(deleted ⇒ StorageIOResult.Success(path, length(deleted)): StorageIOResult) .getOrElse(StorageIOResult.Failure(path, StorageException.NotFound(path))) } .fold(Seq.empty[StorageIOResult])(_ :+ _) - .map(results ⇒ StorageUtils.foldIOResultsIgnoreErrors(results:_*)) + .map(results ⇒ StorageUtils.foldIOResultsIgnoreErrors(results: _*)) .toMat(Sink.head)(Keep.right) } } diff --git a/core/src/main/scala/com/karasiq/shadowcloud/storage/inmem/JVMHeapHealthProvider.scala b/core/src/main/scala/com/karasiq/shadowcloud/storage/inmem/JVMHeapHealthProvider.scala index 41391f0d..66a7d544 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/storage/inmem/JVMHeapHealthProvider.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/storage/inmem/JVMHeapHealthProvider.scala @@ -8,9 +8,8 @@ import com.karasiq.shadowcloud.storage.props.StorageProps.Quota import scala.concurrent.Future -private[storage] final class JVMHeapHealthProvider(dataIterator: () ⇒ Iterator[ByteString], - quota: StorageProps.Quota = StorageProps.Quota.empty) - extends StorageHealthProvider { +private[storage] final class JVMHeapHealthProvider(dataIterator: () ⇒ Iterator[ByteString], quota: StorageProps.Quota = StorageProps.Quota.empty) + extends StorageHealthProvider { def health: Future[StorageHealth] = { val total = Quota.limitTotalSpace(quota, sys.runtime.maxMemory()) diff --git a/core/src/main/scala/com/karasiq/shadowcloud/storage/internal/DefaultIndexRepositoryStreams.scala b/core/src/main/scala/com/karasiq/shadowcloud/storage/internal/DefaultIndexRepositoryStreams.scala index 0c42469c..783d7962 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/storage/internal/DefaultIndexRepositoryStreams.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/storage/internal/DefaultIndexRepositoryStreams.scala @@ -20,10 +20,12 @@ private[storage] final class DefaultIndexRepositoryStreams( extends IndexRepositoryStreams { def write[Key](repository: Repository[Key]): Flow[(Key, IndexData), IndexIOResult[Key], NotUsed] = { - Flow[(Key, IndexData)].flatMapMerge(breadth, { - case (key, value) ⇒ + Flow[(Key, IndexData)].flatMapMerge( + breadth, + { case (key, value) ⇒ Source.single(value).via(writeAndReturn(repository, key)) - }) + } + ) } def read[Key](repository: Repository[Key]): Flow[Key, IndexIOResult[Key], NotUsed] = { @@ -34,7 +36,8 @@ private[storage] final class DefaultIndexRepositoryStreams( if (immutable) return Flow.fromSinkAndSource(Sink.ignore, Source.failed(new IllegalStateException("Storage is immutable"))) Flow[Key].flatMapMerge( - breadth, { key ⇒ + breadth, + { key ⇒ val graph = GraphDSL.create(repository.delete) { implicit builder ⇒ deleteSink ⇒ import GraphDSL.Implicits._ val broadcast = builder.add(Broadcast[Key](2)) @@ -54,11 +57,10 @@ private[storage] final class DefaultIndexRepositoryStreams( import GraphDSL.Implicits._ val broadcast = builder.add(Broadcast[IndexData](2)) val compose = builder.add(ZipWith((diff: IndexData, result: Future[StorageIOResult]) ⇒ (key, diff, result))) - val unwrap = builder.add(Flow[(Key, IndexData, Future[StorageIOResult])].flatMapConcat { - case (key, diff, future) ⇒ - Source - .future(StorageUtils.wrapFuture(repository.toString, future)) - .map(IndexIOResult(key, diff, _)) + val unwrap = builder.add(Flow[(Key, IndexData, Future[StorageIOResult])].flatMapConcat { case (key, diff, future) ⇒ + Source + .future(StorageUtils.wrapFuture(repository.toString, future)) + .map(IndexIOResult(key, diff, _)) }) broadcast ~> compose.in0 broadcast ~> writeFlow ~> repository @@ -79,11 +81,10 @@ private[storage] final class DefaultIndexRepositoryStreams( val graph = GraphDSL.create(repository.read(key)) { implicit builder ⇒ repository ⇒ import GraphDSL.Implicits._ val compose = builder.add(ZipWith((diff: IndexData, result: Future[StorageIOResult]) ⇒ (key, diff, result))) - val unwrap = builder.add(Flow[(Key, IndexData, Future[StorageIOResult])].flatMapConcat { - case (key, diff, future) ⇒ - Source - .future(StorageUtils.wrapFuture(repository.toString, future)) - .map(IndexIOResult(key, diff, _)) + val unwrap = builder.add(Flow[(Key, IndexData, Future[StorageIOResult])].flatMapConcat { case (key, diff, future) ⇒ + Source + .future(StorageUtils.wrapFuture(repository.toString, future)) + .map(IndexIOResult(key, diff, _)) }) repository.out ~> readFlow ~> compose.in0 builder.materializedValue ~> compose.in1 diff --git a/core/src/main/scala/com/karasiq/shadowcloud/storage/replication/ChunkAvailability.scala b/core/src/main/scala/com/karasiq/shadowcloud/storage/replication/ChunkAvailability.scala index 04f8004f..b39f57cf 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/storage/replication/ChunkAvailability.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/storage/replication/ChunkAvailability.scala @@ -3,14 +3,16 @@ package com.karasiq.shadowcloud.storage.replication import com.karasiq.shadowcloud.index.utils.HasEmpty import com.karasiq.shadowcloud.model.StorageId -case class ChunkAvailability(hasChunk: Set[String] = Set.empty, - writingChunk: Set[String] = Set.empty, - writeFailed: Set[String] = Set.empty, - readFailed: Set[String] = Set.empty) extends HasEmpty { +case class ChunkAvailability( + hasChunk: Set[String] = Set.empty, + writingChunk: Set[String] = Set.empty, + writeFailed: Set[String] = Set.empty, + readFailed: Set[String] = Set.empty +) extends HasEmpty { def contains(storageId: StorageId): Boolean = { hasChunk.contains(storageId) || writingChunk.contains(storageId) || - writeFailed.contains(storageId) || readFailed.contains(storageId) + writeFailed.contains(storageId) || readFailed.contains(storageId) } def isEmpty: Boolean = { @@ -52,4 +54,4 @@ case class ChunkAvailability(hasChunk: Set[String] = Set.empty, object ChunkAvailability { val empty = ChunkAvailability() -} \ No newline at end of file +} diff --git a/core/src/main/scala/com/karasiq/shadowcloud/storage/replication/ChunkStatusProvider.scala b/core/src/main/scala/com/karasiq/shadowcloud/storage/replication/ChunkStatusProvider.scala index 80fe11a3..451c6bad 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/storage/replication/ChunkStatusProvider.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/storage/replication/ChunkStatusProvider.scala @@ -10,12 +10,15 @@ object ChunkStatusProvider { object WriteStatus { case class Pending(affinity: ChunkWriteAffinity) extends WriteStatus - case object Finished extends WriteStatus + case object Finished extends WriteStatus } - case class ChunkStatus(writeStatus: WriteStatus, chunk: Chunk, - availability: ChunkAvailability = ChunkAvailability.empty, - waitingChunk: Set[ActorRef] = Set.empty) { + case class ChunkStatus( + writeStatus: WriteStatus, + chunk: Chunk, + availability: ChunkAvailability = ChunkAvailability.empty, + waitingChunk: Set[ActorRef] = Set.empty + ) { def finished: ChunkStatus = { copy(writeStatus = WriteStatus.Finished, chunk = chunk.withoutData) } diff --git a/core/src/main/scala/com/karasiq/shadowcloud/storage/replication/RegionStorageProvider.scala b/core/src/main/scala/com/karasiq/shadowcloud/storage/replication/RegionStorageProvider.scala index 6268f377..476c7d1f 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/storage/replication/RegionStorageProvider.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/storage/replication/RegionStorageProvider.scala @@ -9,8 +9,7 @@ import com.karasiq.shadowcloud.storage.props.StorageProps import com.karasiq.shadowcloud.storage.replication.RegionStorageProvider.RegionStorage object RegionStorageProvider { - case class RegionStorage(id: StorageId, props: StorageProps, config: StorageConfig, - dispatcher: ActorRef, health: StorageHealth) + case class RegionStorage(id: StorageId, props: StorageProps, config: StorageConfig, dispatcher: ActorRef, health: StorageHealth) } trait RegionStorageProvider { diff --git a/core/src/main/scala/com/karasiq/shadowcloud/storage/utils/IndexRepositoryStreams.scala b/core/src/main/scala/com/karasiq/shadowcloud/storage/utils/IndexRepositoryStreams.scala index 3c1b9f6d..1f61e609 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/storage/utils/IndexRepositoryStreams.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/storage/utils/IndexRepositoryStreams.scala @@ -21,15 +21,16 @@ private[shadowcloud] trait IndexRepositoryStreams { } private[shadowcloud] object IndexRepositoryStreams { - def create(breadth: Int, writeFlow: Flow[IndexData, ByteString, _], - readFlow: Flow[ByteString, IndexData, _], immutable: Boolean = false)(implicit ec: ExecutionContext): IndexRepositoryStreams = { + def create(breadth: Int, writeFlow: Flow[IndexData, ByteString, _], readFlow: Flow[ByteString, IndexData, _], immutable: Boolean = false)(implicit + ec: ExecutionContext + ): IndexRepositoryStreams = { new DefaultIndexRepositoryStreams(breadth, writeFlow, readFlow, immutable) } def apply(regionId: RegionId, storageConfig: StorageConfig, actorSystem: ActorSystem): IndexRepositoryStreams = { import actorSystem.dispatcher implicit val sc = ShadowCloud(actorSystem) - val index = IndexProcessingStreams(regionId) + val index = IndexProcessingStreams(regionId) create(3, index.preWrite(storageConfig), index.postRead, storageConfig.immutable) } diff --git a/core/src/main/scala/com/karasiq/shadowcloud/storage/utils/StoragePluginBuilder.scala b/core/src/main/scala/com/karasiq/shadowcloud/storage/utils/StoragePluginBuilder.scala index ef30dcc6..479cd171 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/storage/utils/StoragePluginBuilder.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/storage/utils/StoragePluginBuilder.scala @@ -27,12 +27,14 @@ object StoragePluginBuilder { } } -case class StoragePluginBuilder(storageId: StorageId, - props: StorageProps, - index: Option[CategorizedRepository[RegionId, SequenceNr]] = None, - chunks: Option[CategorizedRepository[RegionId, ChunkId]] = None, - health: Option[StorageHealthProvider] = None, - lifecycleHook: Option[LifecycleHook] = None) { +case class StoragePluginBuilder( + storageId: StorageId, + props: StorageProps, + index: Option[CategorizedRepository[RegionId, SequenceNr]] = None, + chunks: Option[CategorizedRepository[RegionId, ChunkId]] = None, + health: Option[StorageHealthProvider] = None, + lifecycleHook: Option[LifecycleHook] = None +) { def withIndex(repository: CategorizedRepository[RegionId, SequenceNr]): StoragePluginBuilder = { copy(index = Some(repository)) @@ -70,7 +72,7 @@ case class StoragePluginBuilder(storageId: StorageId, copy(health = Some(healthProvider)) } - def withHealthF(f: () => Future[StorageHealth]): StoragePluginBuilder = { + def withHealthF(f: () ⇒ Future[StorageHealth]): StoragePluginBuilder = { withHealth(new StorageHealthProvider { override def health: Future[StorageHealth] = f() }) @@ -87,9 +89,10 @@ case class StoragePluginBuilder(storageId: StorageId, // require(health.nonEmpty, "Health provider not provided") val indexSynchronizer = context.actorOf(StorageIndex.props(storageId, props, index.get), "index") - val chunkIO = context.actorOf(ChunkIODispatcher.props(storageId, props, chunks.get), "chunks") - val healthProvider = StorageHealthProvider.applyQuota(health.getOrElse(StorageHealthProvider.unlimited), props.quota) - val storageDispatcher = context.actorOf(StorageDispatcher.props(storageId, props, indexSynchronizer, chunkIO, healthProvider, lifecycleHook), "storageDispatcher") + val chunkIO = context.actorOf(ChunkIODispatcher.props(storageId, props, chunks.get), "chunks") + val healthProvider = StorageHealthProvider.applyQuota(health.getOrElse(StorageHealthProvider.unlimited), props.quota) + val storageDispatcher = + context.actorOf(StorageDispatcher.props(storageId, props, indexSynchronizer, chunkIO, healthProvider, lifecycleHook), "storageDispatcher") context.watch(indexSynchronizer) context.watch(chunkIO) context.watch(storageDispatcher) diff --git a/core/src/main/scala/com/karasiq/shadowcloud/streams/chunk/BytesBuffer.scala b/core/src/main/scala/com/karasiq/shadowcloud/streams/chunk/BytesBuffer.scala index 130eba37..2413a037 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/streams/chunk/BytesBuffer.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/streams/chunk/BytesBuffer.scala @@ -1,3 +1 @@ package com.karasiq.shadowcloud.streams.chunk - - diff --git a/core/src/main/scala/com/karasiq/shadowcloud/streams/chunk/ChunkKeyStream.scala b/core/src/main/scala/com/karasiq/shadowcloud/streams/chunk/ChunkKeyStream.scala index 019f60ad..ce3e02c8 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/streams/chunk/ChunkKeyStream.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/streams/chunk/ChunkKeyStream.scala @@ -9,9 +9,7 @@ import com.karasiq.shadowcloud.model.crypto._ import com.karasiq.shadowcloud.providers.CryptoModuleRegistry private[shadowcloud] object ChunkKeyStream { - def apply(cryptoModules: CryptoModuleRegistry, - method: EncryptionMethod, - maxKeyReuse: Int = 256): ChunkKeyStream = { + def apply(cryptoModules: CryptoModuleRegistry, method: EncryptionMethod, maxKeyReuse: Int = 256): ChunkKeyStream = { new ChunkKeyStream(cryptoModules, method, maxKeyReuse) } @@ -42,17 +40,17 @@ private[shadowcloud] object ChunkKeyStream { } private[shadowcloud] final class ChunkKeyStream(cryptoModules: CryptoModuleRegistry, method: EncryptionMethod, maxKeyReuse: Int) - extends GraphStage[SourceShape[EncryptionParameters]] { + extends GraphStage[SourceShape[EncryptionParameters]] { val outlet = Outlet[EncryptionParameters]("ChunkKeyStream.out") - val shape = SourceShape(outlet) + val shape = SourceShape(outlet) def createLogic(inheritedAttributes: Attributes) = new GraphStageLogic(shape) with OutHandler { - private[this] val encryptionModule = cryptoModules.encryptionModule(method) - private[this] var secureRandom: SecureRandom = _ + private[this] val encryptionModule = cryptoModules.encryptionModule(method) + private[this] var secureRandom: SecureRandom = _ private[this] var keyParameters: EncryptionParameters = _ - private[this] var encryptedCount: Int = _ - private[this] var changeKeyIn: Int = _ + private[this] var encryptedCount: Int = _ + private[this] var changeKeyIn: Int = _ private[this] def resetParametersAndCounter(): Unit = { // TODO: Log key changes this.keyParameters = encryptionModule.createParameters() diff --git a/core/src/main/scala/com/karasiq/shadowcloud/streams/chunk/ChunkProcessingStreams.scala b/core/src/main/scala/com/karasiq/shadowcloud/streams/chunk/ChunkProcessingStreams.scala index 87179b86..08199b6b 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/streams/chunk/ChunkProcessingStreams.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/streams/chunk/ChunkProcessingStreams.scala @@ -14,8 +14,8 @@ import com.karasiq.shadowcloud.streams.file.FileIndexer import scala.concurrent.{ExecutionContext, Future} object ChunkProcessingStreams { - def apply(cryptoModules: CryptoModuleRegistry, chunks: ChunksConfig, crypto: CryptoConfig, parallelism: ParallelismConfig)( - implicit ec: ExecutionContext + def apply(cryptoModules: CryptoModuleRegistry, chunks: ChunksConfig, crypto: CryptoConfig, parallelism: ParallelismConfig)(implicit + ec: ExecutionContext ): ChunkProcessingStreams = { new ChunkProcessingStreams(cryptoModules, chunks, crypto, parallelism) } @@ -105,8 +105,10 @@ final class ChunkProcessingStreams(cryptoModules: CryptoModuleRegistry, chunks: val hasher = cryptoModules.hashingModule(chunk.checksum.method) - if ((chunk.data.plain.nonEmpty && chunk.checksum.size != chunk.data.plain.length) || - (chunk.data.encrypted.nonEmpty && chunk.checksum.encSize != chunk.data.encrypted.length)) { + if ( + (chunk.data.plain.nonEmpty && chunk.checksum.size != chunk.data.plain.length) || + (chunk.data.encrypted.nonEmpty && chunk.checksum.encSize != chunk.data.encrypted.length) + ) { throw SCExceptions.ChunkVerifyError(chunk, new IllegalArgumentException("Chunk sizes not match")) } @@ -122,7 +124,7 @@ final class ChunkProcessingStreams(cryptoModules: CryptoModuleRegistry, chunks: encHashing: HashingMethod = crypto.hashing.chunksEncrypted ): ChunkFlow = { generateKeys(encryption) - // .buffer(10, OverflowStrategy.backpressure) + // .buffer(10, OverflowStrategy.backpressure) .via(encrypt) .via(createHashes(hashing, encHashing)) } diff --git a/core/src/main/scala/com/karasiq/shadowcloud/streams/chunk/ChunkRanges.scala b/core/src/main/scala/com/karasiq/shadowcloud/streams/chunk/ChunkRanges.scala index 21aebfcf..15239ccd 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/streams/chunk/ChunkRanges.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/streams/chunk/ChunkRanges.scala @@ -34,7 +34,7 @@ object ChunkRanges { data } else { val start = math.max(this.start, 0L) - val end = math.min(this.end, data.length) + val end = math.min(this.end, data.length) data.slice(start.toInt, end.toInt) // data.drop(start.toInt).take((end - start).toInt) } } @@ -42,7 +42,7 @@ object ChunkRanges { object Range { implicit val ordering = Ordering.by((r: Range) ⇒ (r.start, r.end)) - + implicit def fromScalaRange(range: scala.Range): Range = { Range(range.start, if (range.isInclusive) range.end + 1 else range.end) } @@ -119,7 +119,7 @@ object ChunkRanges { for (chunk ← chunkStream) { val chunkSize = chunk.checksum.size - val range = Range(position, position + chunkSize) + val range = Range(position, position + chunkSize) rangedChunks += (chunk → range) position += chunkSize } @@ -128,8 +128,11 @@ object ChunkRanges { def mapChunkStream(ranges: RangeList, chunkStream: Seq[Chunk]): Seq[(Chunk, RangeList)] = { @tailrec - def groupRanges(ranges: Seq[(Chunk, Range)], currentChunk: Option[(Chunk, RangeList)], - result: Seq[(Chunk, RangeList)]): Seq[(Chunk, RangeList)] = ranges match { + def groupRanges( + ranges: Seq[(Chunk, Range)], + currentChunk: Option[(Chunk, RangeList)], + result: Seq[(Chunk, RangeList)] + ): Seq[(Chunk, RangeList)] = ranges match { case (chunk, range) +: tail ⇒ if (currentChunk.exists(_._1 == chunk)) { groupRanges(tail, currentChunk.map { case (c, ranges) ⇒ (c, ranges + range) }, result) diff --git a/core/src/main/scala/com/karasiq/shadowcloud/streams/chunk/ChunkSplitter.scala b/core/src/main/scala/com/karasiq/shadowcloud/streams/chunk/ChunkSplitter.scala index 307b9a89..afb2e9f3 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/streams/chunk/ChunkSplitter.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/streams/chunk/ChunkSplitter.scala @@ -1,7 +1,5 @@ package com.karasiq.shadowcloud.streams.chunk - - import akka.NotUsed import akka.stream.scaladsl.Flow import akka.stream.stage.{GraphStage, GraphStageLogic, InHandler, OutHandler} @@ -16,15 +14,14 @@ private[shadowcloud] object ChunkSplitter { } } -/** - * Splits input data to fixed size chunks +/** Splits input data to fixed size chunks * @param chunkSize Output chunk size */ private[shadowcloud] final class ChunkSplitter(chunkSize: Int) extends GraphStage[FlowShape[ByteString, Chunk]] { require(chunkSize > 0) - val inBytes = Inlet[ByteString]("FileSplitter.inBytes") + val inBytes = Inlet[ByteString]("FileSplitter.inBytes") val outChunks = Outlet[Chunk]("FileSplitter.outChunks") - val shape = FlowShape(inBytes, outChunks) + val shape = FlowShape(inBytes, outChunks) def createLogic(inheritedAttributes: Attributes) = new GraphStageLogic(shape) with InHandler with OutHandler { private[this] var buffer = ByteString.empty diff --git a/core/src/main/scala/com/karasiq/shadowcloud/streams/file/FileIndexer.scala b/core/src/main/scala/com/karasiq/shadowcloud/streams/file/FileIndexer.scala index 4c8cd9f6..42b9a0be 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/streams/file/FileIndexer.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/streams/file/FileIndexer.scala @@ -18,16 +18,17 @@ import scala.util.{Failure, Success, Try} private[shadowcloud] object FileIndexer { case class Result(checksum: Checksum, chunks: Seq[Chunk], ioResult: IOResult) - def apply(registry: CryptoModuleRegistry, plainHashing: HashingMethod = HashingMethod.default, - encryptedHashing: HashingMethod = HashingMethod.default): Sink[Chunk, Future[Result]] = { + def apply( + registry: CryptoModuleRegistry, + plainHashing: HashingMethod = HashingMethod.default, + encryptedHashing: HashingMethod = HashingMethod.default + ): Sink[Chunk, Future[Result]] = { Sink.fromGraph(new FileIndexer(registry, plainHashing, encryptedHashing)).named("fileIndexer") } } -private[shadowcloud] final class FileIndexer(cryptoModules: CryptoModuleRegistry, - plainHashing: HashingMethod, - encryptedHashing: HashingMethod) - extends GraphStageWithMaterializedValue[SinkShape[Chunk], Future[FileIndexer.Result]] { +private[shadowcloud] final class FileIndexer(cryptoModules: CryptoModuleRegistry, plainHashing: HashingMethod, encryptedHashing: HashingMethod) + extends GraphStageWithMaterializedValue[SinkShape[Chunk], Future[FileIndexer.Result]] { val inlet = Inlet[Chunk]("FileIndexer.in") val shape = SinkShape(inlet) @@ -46,9 +47,9 @@ private[shadowcloud] final class FileIndexer(cryptoModules: CryptoModuleRegistry .map(CryptoModuleRegistry.streamHashingModule(cryptoModules, _)) .map(_.createStreamer()) - private[this] var plainSize = 0L + private[this] var plainSize = 0L private[this] var encryptedSize = 0L - private[this] val chunks = Vector.newBuilder[Chunk] + private[this] val chunks = Vector.newBuilder[Chunk] private[this] def update(chunk: Chunk): Unit = { hasher.foreach { module ⇒ @@ -67,9 +68,14 @@ private[shadowcloud] final class FileIndexer(cryptoModules: CryptoModuleRegistry } private[this] def finish(status: Try[Done]): Unit = { - val checksum = Checksum(plainHashing, encryptedHashing, - plainSize, hasher.fold(ByteString.empty)(_.finish()), - encryptedSize, encHasher.fold(ByteString.empty)(_.finish())) + val checksum = Checksum( + plainHashing, + encryptedHashing, + plainSize, + hasher.fold(ByteString.empty)(_.finish()), + encryptedSize, + encHasher.fold(ByteString.empty)(_.finish()) + ) val indexedFile = FileIndexer.Result(checksum, chunks.result(), IOResult(plainSize, status)) promise.trySuccess(indexedFile) @@ -86,7 +92,6 @@ private[shadowcloud] final class FileIndexer(cryptoModules: CryptoModuleRegistry completeStage() } - override def onUpstreamFailure(ex: Throwable): Unit = { finish(Failure(ex)) super.onUpstreamFailure(ex) diff --git a/core/src/main/scala/com/karasiq/shadowcloud/streams/file/FileStreams.scala b/core/src/main/scala/com/karasiq/shadowcloud/streams/file/FileStreams.scala index e4c87e72..1d83f1a3 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/streams/file/FileStreams.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/streams/file/FileStreams.scala @@ -15,8 +15,8 @@ import com.karasiq.shadowcloud.streams.utils.AkkaStreamUtils import scala.concurrent.Future object FileStreams { - def apply(regionStreams: RegionStreams, chunkProcessing: ChunkProcessingStreams, supervisorOps: RegionSupervisorOps)( - implicit m: Materializer + def apply(regionStreams: RegionStreams, chunkProcessing: ChunkProcessingStreams, supervisorOps: RegionSupervisorOps)(implicit + m: Materializer ): FileStreams = { new FileStreams(regionStreams, chunkProcessing, supervisorOps) } @@ -93,26 +93,25 @@ final class FileStreams(regionStreams: RegionStreams, chunkProcessing: ChunkProc Flow[ByteString] .via(AkkaStreamUtils.extractUpstream) .zip(Source.future(supervisorOps.getRegionConfig(regionId))) - .flatMapConcat { - case (stream, config) ⇒ - val matSink = Flow - .fromGraph(chunkProcessing.split(config.chunkSize.getOrElse(chunkProcessing.defaultChunkSize))) - .via(chunkProcessing.beforeWrite()) - // .map(c ⇒ c.copy(data = c.data.copy(plain = ByteString.empty))) // Memory optimization - .map((regionId, _)) - .via(regionStreams.writeChunks) - // .log("chunk-stream-write") - .toMat(chunkProcessing.index())(Keep.right) - - val graph = GraphDSL.create(matSink) { implicit builder ⇒ matSink ⇒ - import GraphDSL.Implicits._ - val extractResult = builder.add(Flow[Future[FileIndexer.Result]].flatMapConcat(Source.future)) - - builder.materializedValue ~> extractResult - FlowShape(matSink.in, extractResult.out) - } - - stream.via(graph) + .flatMapConcat { case (stream, config) ⇒ + val matSink = Flow + .fromGraph(chunkProcessing.split(config.chunkSize.getOrElse(chunkProcessing.defaultChunkSize))) + .via(chunkProcessing.beforeWrite()) + // .map(c ⇒ c.copy(data = c.data.copy(plain = ByteString.empty))) // Memory optimization + .map((regionId, _)) + .via(regionStreams.writeChunks) + // .log("chunk-stream-write") + .toMat(chunkProcessing.index())(Keep.right) + + val graph = GraphDSL.create(matSink) { implicit builder ⇒ matSink ⇒ + import GraphDSL.Implicits._ + val extractResult = builder.add(Flow[Future[FileIndexer.Result]].flatMapConcat(Source.future)) + + builder.materializedValue ~> extractResult + FlowShape(matSink.in, extractResult.out) + } + + stream.via(graph) } .named("writeChunkStream") } diff --git a/core/src/main/scala/com/karasiq/shadowcloud/streams/index/IndexProcessingStreams.scala b/core/src/main/scala/com/karasiq/shadowcloud/streams/index/IndexProcessingStreams.scala index ab73d959..f554be3c 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/streams/index/IndexProcessingStreams.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/streams/index/IndexProcessingStreams.scala @@ -1,6 +1,5 @@ package com.karasiq.shadowcloud.streams.index - import akka.NotUsed import akka.event.Logging import akka.stream._ @@ -56,7 +55,8 @@ final class IndexProcessingStreams(regionId: RegionId)(implicit sc: ShadowCloudE .named("indexPostRead") private[this] object internalStreams { - private[this] val keyChainSource = Source.single(NotUsed) + private[this] val keyChainSource = Source + .single(NotUsed) .flatMapConcat(_ ⇒ Source.future(sc.keys.provider.getKeyChain())) .map(_.forRegion(regionId)) .named("keyChainSource") @@ -94,13 +94,15 @@ final class IndexProcessingStreams(regionId: RegionId)(implicit sc: ShadowCloudE .via(StreamSerialization.deframe(sc.config.serialization.frameLimit)) val compress = Flow[ByteString].flatMapConcat { bytes ⇒ - Source.single(bytes) + Source + .single(bytes) .via(framedCompress) .via(ByteStreams.concat) } val decompress = Flow[ByteString].flatMapConcat { compBytes ⇒ - Source.single(compBytes) + Source + .single(compBytes) .via(framedDecompress) .via(ByteStreams.concat) } @@ -118,14 +120,14 @@ final class IndexProcessingStreams(regionId: RegionId)(implicit sc: ShadowCloudE // Encryption stages // ----------------------------------------------------------------------- private final class IndexEncryptStage(cryptoModules: CryptoModuleRegistry, config: CryptoConfig, keyChain: KeyChain) - extends GraphStage[FlowShape[ByteString, EncryptedIndexData]] { + extends GraphStage[FlowShape[ByteString, EncryptedIndexData]] { - val inlet = Inlet[ByteString]("IndexEncrypt.in") + val inlet = Inlet[ByteString]("IndexEncrypt.in") val outlet = Outlet[EncryptedIndexData]("IndexEncrypt.out") - val shape = FlowShape(inlet, outlet) + val shape = FlowShape(inlet, outlet) def createLogic(inheritedAttributes: Attributes) = new GraphStageLogic(shape) with InHandler with OutHandler { - private[this] val log = Logging(sc.implicits.actorSystem, s"IndexEncryptStage($regionId)") + private[this] val log = Logging(sc.implicits.actorSystem, s"IndexEncryptStage($regionId)") private[this] val encryption = IndexEncryption(cryptoModules, serialization) private[this] def encryptData(data: ByteString): Unit = { @@ -153,14 +155,14 @@ final class IndexProcessingStreams(regionId: RegionId)(implicit sc: ShadowCloudE } private final class IndexDecryptStage(cryptoModules: CryptoModuleRegistry, config: CryptoConfig, keyChain: KeyChain) - extends GraphStage[FlowShape[EncryptedIndexData, ByteString]] { + extends GraphStage[FlowShape[EncryptedIndexData, ByteString]] { - val inlet = Inlet[EncryptedIndexData]("IndexDecrypt.in") + val inlet = Inlet[EncryptedIndexData]("IndexDecrypt.in") val outlet = Outlet[ByteString]("IndexDecrypt.out") - val shape = FlowShape(inlet, outlet) + val shape = FlowShape(inlet, outlet) def createLogic(inheritedAttributes: Attributes) = new GraphStageLogic(shape) with InHandler with OutHandler { - private[this] val log = Logging(sc.implicits.actorSystem, s"IndexDecryptStage($regionId)") + private[this] val log = Logging(sc.implicits.actorSystem, s"IndexDecryptStage($regionId)") private[this] val encryption = IndexEncryption(cryptoModules, serialization) private[this] def pullInlet(): Unit = { @@ -176,11 +178,12 @@ final class IndexProcessingStreams(regionId: RegionId)(implicit sc: ShadowCloudE log.info("Decrypting index frame {} with keys: {}", data.id, keyChain.decKeys) val result = encryption.decrypt(data, keyChain) push(outlet, result) - } catch { case NonFatal(exc) ⇒ - log.error(CryptoException.DecryptError(exc), "Index file decrypt error") - pullInlet() + } catch { + case NonFatal(exc) ⇒ + log.error(CryptoException.DecryptError(exc), "Index file decrypt error") + pullInlet() } - } + } def onPull(): Unit = { pullInlet() diff --git a/core/src/main/scala/com/karasiq/shadowcloud/streams/metadata/MetadataStreams.scala b/core/src/main/scala/com/karasiq/shadowcloud/streams/metadata/MetadataStreams.scala index c77e6798..9da68556 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/streams/metadata/MetadataStreams.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/streams/metadata/MetadataStreams.scala @@ -8,7 +8,7 @@ import akka.stream.scaladsl.{Broadcast, Flow, GraphDSL, Keep, Source, Zip} import akka.util.ByteString import com.karasiq.shadowcloud.compression.StreamCompression import com.karasiq.shadowcloud.config.{MetadataConfig, SerializationConfig} -import com.karasiq.shadowcloud.metadata.Metadata.Tag.{Disposition => MDDisposition} +import com.karasiq.shadowcloud.metadata.Metadata.Tag.{Disposition ⇒ MDDisposition} import com.karasiq.shadowcloud.metadata.{Metadata, MetadataUtils} import com.karasiq.shadowcloud.model._ import com.karasiq.shadowcloud.ops.region.RegionOps @@ -21,33 +21,48 @@ import com.karasiq.shadowcloud.streams.utils.{AkkaStreamUtils, ByteStreams} import scala.concurrent.Future private[shadowcloud] object MetadataStreams { - def apply(regionOps: RegionOps, regionStreams: RegionStreams, fileStreams: FileStreams, - metadataConfig: MetadataConfig, metadataModules: MetadataModuleRegistry, - serializationConfig: SerializationConfig, serialization: SerializationModule): MetadataStreams = { + def apply( + regionOps: RegionOps, + regionStreams: RegionStreams, + fileStreams: FileStreams, + metadataConfig: MetadataConfig, + metadataModules: MetadataModuleRegistry, + serializationConfig: SerializationConfig, + serialization: SerializationModule + ): MetadataStreams = { new MetadataStreams(regionOps, regionStreams, fileStreams, metadataConfig, metadataModules, serializationConfig, serialization) } } -private[shadowcloud] final class MetadataStreams(regionOps: RegionOps, regionStreams: RegionStreams, fileStreams: FileStreams, - metadataConfig: MetadataConfig, metadataModules: MetadataModuleRegistry, - serializationConfig: SerializationConfig, serialization: SerializationModule) { - +private[shadowcloud] final class MetadataStreams( + regionOps: RegionOps, + regionStreams: RegionStreams, + fileStreams: FileStreams, + metadataConfig: MetadataConfig, + metadataModules: MetadataModuleRegistry, + serializationConfig: SerializationConfig, + serialization: SerializationModule +) { + def keys(regionId: RegionId): Source[FileId, NotUsed] = { - Source.future(regionOps.getFolder(regionId, MetadataUtils.MetadataFolder)) + Source + .future(regionOps.getFolder(regionId, MetadataUtils.MetadataFolder)) .recover { case _ ⇒ Folder(MetadataUtils.MetadataFolder) } .mapConcat(_.folders.map(UUID.fromString)) .named("metadataFileKeys") } - + def write(regionId: RegionId, fileId: FileId, disposition: MDDisposition): Flow[Metadata, File, NotUsed] = { - internalStreams.preWriteFile(regionId) + internalStreams + .preWriteFile(regionId) .map((regionId, MetadataUtils.getFilePath(fileId, disposition), _)) .via(regionStreams.createFile) .named("metadataWrite") } def writeAll(regionId: RegionId, fileId: FileId): Flow[Metadata, File, NotUsed] = { - internalStreams.preWriteAll(regionId) + internalStreams + .preWriteAll(regionId) .map { case (disposition, result) ⇒ (regionId, MetadataUtils.getFilePath(fileId, disposition), result) } .via(regionStreams.createFile) .named("metadataWriteAll") @@ -73,20 +88,21 @@ private[shadowcloud] final class MetadataStreams(regionOps: RegionOps, regionStr val graph = GraphDSL.create() { implicit builder ⇒ import GraphDSL.Implicits._ - val bytesInput = builder.add(Broadcast[ByteString](2)) - val extractStream = builder.add(Flow[ByteString].async.via(AkkaStreamUtils.extractUpstream)) + val bytesInput = builder.add(Broadcast[ByteString](2)) + val extractStream = builder.add(Flow[ByteString].async.via(AkkaStreamUtils.extractUpstream)) val getContentType = builder.add(Flow[ByteString].async.via(MimeDetectorStream(metadataModules, fileName, metadataConfig.mimeProbeSize))) val zipStreamAndMime = builder.add(Zip[Source[ByteString, NotUsed], String]) - val parseMetadata = builder.add(Flow[(Source[ByteString, NotUsed], String)] - .flatMapConcat { case (bytes, contentType) ⇒ - if (metadataModules.canParse(fileName, contentType)) { - bytes.via(metadataModules.parseMetadata(fileName, contentType)) - } else { - bytes.via(AkkaStreamUtils.cancelUpstream(Source.empty)) // Bytes stream should be cancelled + val parseMetadata = builder.add( + Flow[(Source[ByteString, NotUsed], String)] + .flatMapConcat { case (bytes, contentType) ⇒ + if (metadataModules.canParse(fileName, contentType)) { + bytes.via(metadataModules.parseMetadata(fileName, contentType)) + } else { + bytes.via(AkkaStreamUtils.cancelUpstream(Source.empty)) // Bytes stream should be cancelled + } } - } ) bytesInput ~> getContentType @@ -104,9 +120,11 @@ private[shadowcloud] final class MetadataStreams(regionOps: RegionOps, regionStr .named("metadataCreate") } - def writeFileAndMetadata(regionId: RegionId, path: Path, - metadataSizeLimit: Long = metadataConfig.fileSizeLimit - ): Flow[ByteString, (File, Seq[File]), NotUsed] = { + def writeFileAndMetadata( + regionId: RegionId, + path: Path, + metadataSizeLimit: Long = metadataConfig.fileSizeLimit + ): Flow[ByteString, (File, Seq[File]), NotUsed] = { // Writes the chunk stream before actual file path is known val writeStream = Flow[ByteString] .via(fileStreams.write(regionId, path)) @@ -116,28 +134,29 @@ private[shadowcloud] final class MetadataStreams(regionOps: RegionOps, regionStr .buffer(5, OverflowStrategy.backpressure) // Buffer byte chunks .via(create(path.name, metadataSizeLimit)) .async - // .buffer(10, OverflowStrategy.backpressure) // Buffer metadatas + // .buffer(10, OverflowStrategy.backpressure) // Buffer metadatas val graph = GraphDSL.create(writeStream, createMetadataStream)(Keep.none) { implicit builder ⇒ (writeFile, createMetadata) ⇒ import GraphDSL.Implicits._ val bytesInput = builder.add(Broadcast[ByteString](2)) - val fileInput = builder.add(Broadcast[File](2)) + val fileInput = builder.add(Broadcast[File](2)) - val writeMetadataChunks = builder.add(internalStreams.preWriteAll(regionId)) + val writeMetadataChunks = builder.add(internalStreams.preWriteAll(regionId)) val extractMetadataSource = builder.add(Flow[(MDDisposition, FileIndexer.Result)].via(AkkaStreamUtils.extractUpstream)) - val zipSourceAndFile = builder.add(Zip[Source[(MDDisposition, FileIndexer.Result), NotUsed], File]) - val writeMetadata = builder.add(Flow[(Source[(MDDisposition, FileIndexer.Result), NotUsed], File)] - .flatMapConcat { case (metadataIn, file) ⇒ - metadataIn.flatMapConcat { case (disposition, chunkStream) ⇒ - val path = MetadataUtils.getFilePath(file.id, disposition) - val newFile = File.create(path, chunkStream.checksum, chunkStream.chunks) - Source.future(regionOps.createFile(regionId, newFile)) + val zipSourceAndFile = builder.add(Zip[Source[(MDDisposition, FileIndexer.Result), NotUsed], File]) + val writeMetadata = builder.add( + Flow[(Source[(MDDisposition, FileIndexer.Result), NotUsed], File)] + .flatMapConcat { case (metadataIn, file) ⇒ + metadataIn.flatMapConcat { case (disposition, chunkStream) ⇒ + val path = MetadataUtils.getFilePath(file.id, disposition) + val newFile = File.create(path, chunkStream.checksum, chunkStream.chunks) + Source.future(regionOps.createFile(regionId, newFile)) + } } - } - .log("metadata-files") - .fold(Vector.empty[File])(_ :+ _) - .recover { case _ ⇒ Vector.empty } + .log("metadata-files") + .fold(Vector.empty[File])(_ :+ _) + .recover { case _ ⇒ Vector.empty } ) val zipFileAndMetadata = builder.add(Zip[File, Seq[File]]) @@ -169,9 +188,9 @@ private[shadowcloud] final class MetadataStreams(regionOps: RegionOps, regionStr .flatMapConcat { stream ⇒ val graph = GraphDSL.create() { implicit builder ⇒ import GraphDSL.Implicits._ - val broadcast = builder.add(Broadcast[Metadata](2)) - val extractDisposition = builder.add(Flow[Metadata].map(md ⇒ MetadataUtils.getDisposition(md.tag))) - val writeChunks = builder.add(preWriteFile(regionId)) + val broadcast = builder.add(Broadcast[Metadata](2)) + val extractDisposition = builder.add(Flow[Metadata].map(md ⇒ MetadataUtils.getDisposition(md.tag))) + val writeChunks = builder.add(preWriteFile(regionId)) val zipDispositionAndChunks = builder.add(Zip[MDDisposition, FileIndexer.Result]()) broadcast ~> extractDisposition @@ -190,7 +209,8 @@ private[shadowcloud] final class MetadataStreams(regionOps: RegionOps, regionStr } def readMetadataFile(regionId: RegionId, fileId: FileId, disposition: MDDisposition) = { - fileStreams.readMostRecent(regionId, MetadataUtils.getFilePath(fileId, disposition)) + fileStreams + .readMostRecent(regionId, MetadataUtils.getFilePath(fileId, disposition)) .via(StreamCompression.decompress) .via(StreamSerialization.deserializeFramed[Metadata](serialization, serializationConfig.frameLimit)) .recoverWithRetries(1, { case _ ⇒ Source.empty }) diff --git a/core/src/main/scala/com/karasiq/shadowcloud/streams/metadata/MimeDetectorStream.scala b/core/src/main/scala/com/karasiq/shadowcloud/streams/metadata/MimeDetectorStream.scala index a657d5b6..d30f1233 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/streams/metadata/MimeDetectorStream.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/streams/metadata/MimeDetectorStream.scala @@ -12,7 +12,8 @@ import com.karasiq.shadowcloud.metadata.MimeDetector object MimeDetectorStream { def apply(detector: MimeDetector, fileName: String, probeSize: Int): Flow[ByteString, String, NotUsed] = { val defaultMimeSource = Source.single(MimeDetector.DefaultMime) - Flow.fromGraph(new MimeDetectorStream(detector, fileName, probeSize)) + Flow + .fromGraph(new MimeDetectorStream(detector, fileName, probeSize)) .recoverWithRetries(1, { case _ ⇒ defaultMimeSource }) .orElse(defaultMimeSource) .withAttributes(ActorAttributes.dispatcher(SCDispatchers.metadata)) @@ -20,19 +21,18 @@ object MimeDetectorStream { } } -private final class MimeDetectorStream(detector: MimeDetector, fileName: String, probeSize: Int) - extends GraphStage[FlowShape[ByteString, String]] { +private final class MimeDetectorStream(detector: MimeDetector, fileName: String, probeSize: Int) extends GraphStage[FlowShape[ByteString, String]] { - val inBytes = Inlet[ByteString]("MimeDetectorStream.in") + val inBytes = Inlet[ByteString]("MimeDetectorStream.in") val outContentType = Outlet[String]("MimeDetectorStream.out") - val shape = FlowShape(inBytes, outContentType) + val shape = FlowShape(inBytes, outContentType) def createLogic(inheritedAttributes: Attributes) = new GraphStageLogic(shape) with InHandler with OutHandler { private[this] var buffer = ByteString.empty private[this] def pushContentType(): Unit = { if (buffer.length >= probeSize || isClosed(inBytes)) { - val contentType = detector.getMimeType(fileName, buffer/*.take(probeSize)*/) + val contentType = detector.getMimeType(fileName, buffer /*.take(probeSize)*/ ) buffer = ByteString.empty contentType.foreach(push(outContentType, _)) completeStage() @@ -57,4 +57,4 @@ private final class MimeDetectorStream(detector: MimeDetector, fileName: String, setHandlers(inBytes, outContentType, this) } -} \ No newline at end of file +} diff --git a/core/src/main/scala/com/karasiq/shadowcloud/streams/region/RegionRepairStream.scala b/core/src/main/scala/com/karasiq/shadowcloud/streams/region/RegionRepairStream.scala index 26635cd0..5ec89a02 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/streams/region/RegionRepairStream.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/streams/region/RegionRepairStream.scala @@ -64,16 +64,20 @@ object RegionRepairStream { Flow[ChunkStatus] .map(status ⇒ status → createNewAffinity(status, request.strategy)) .filterNot { case (status, newAffinity) ⇒ newAffinity.exists(_.isFinished(status)) } - .flatMapMerge(parallelism.read, { - case (status, newAffinity) ⇒ + .flatMapMerge( + parallelism.read, + { case (status, newAffinity) ⇒ futureWithRetry(regionOps.readChunkEncrypted(request.regionId, status.chunk).map(_ → newAffinity)) - }) + } + ) .log("region-repair-read", chunk ⇒ s"${chunk._1.hashString} at ${request.regionId}") .via(ByteStreams.bufferT(_._1.data.encrypted.length, bufferSize)) - .flatMapMerge(parallelism.write, { - case (chunk, newAffinity) ⇒ + .flatMapMerge( + parallelism.write, + { case (chunk, newAffinity) ⇒ futureWithRetry(regionOps.rewriteChunk(request.regionId, chunk, newAffinity)) - }) + } + ) .log("region-repair-write", chunk ⇒ s"${chunk.hashString} at ${request.regionId}") // .withAttributes(ActorAttributes.logLevels(onElement = Logging.InfoLevel)) .map(_.withoutData) diff --git a/core/src/main/scala/com/karasiq/shadowcloud/streams/region/RegionStreams.scala b/core/src/main/scala/com/karasiq/shadowcloud/streams/region/RegionStreams.scala index 7095a144..5fe2f985 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/streams/region/RegionStreams.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/streams/region/RegionStreams.scala @@ -40,12 +40,11 @@ final class RegionStreams(regionOps: RegionOps, parallelism: ParallelismConfig, .named("readChunksNonBuffered") val findFiles = Flow[(RegionId, Path)] - .mapAsync(parallelism.query) { - case (regionId, path) ⇒ - regionOps - .getFiles(regionId, path) - .map((path, _)) - .recover { case error if SCException.isNotFound(error) ⇒ (path, Set.empty[File]) } + .mapAsync(parallelism.query) { case (regionId, path) ⇒ + regionOps + .getFiles(regionId, path) + .map((path, _)) + .recover { case error if SCException.isNotFound(error) ⇒ (path, Set.empty[File]) } } .named("findFiles") @@ -54,18 +53,16 @@ final class RegionStreams(regionOps: RegionOps, parallelism: ParallelismConfig, .named("findFile") val getFolder = Flow[(RegionId, Path)] - .mapAsync(parallelism.query) { - case (regionId, path) ⇒ - regionOps.getFolder(regionId, path) + .mapAsync(parallelism.query) { case (regionId, path) ⇒ + regionOps.getFolder(regionId, path) } .named("getFolder") val createFile: Flow[(RegionId, Path, FileIndexer.Result), File, NotUsed] = { Flow[(String, Path, FileIndexer.Result)] - .flatMapConcat { - case (regionId, path, result) ⇒ - val future = regionOps.createFile(regionId, File.create(path, result.checksum, result.chunks)) - Source.future(future) + .flatMapConcat { case (regionId, path, result) ⇒ + val future = regionOps.createFile(regionId, File.create(path, result.checksum, result.chunks)) + Source.future(future) } .named("createFile") } diff --git a/core/src/main/scala/com/karasiq/shadowcloud/ui/ChallengeHub.scala b/core/src/main/scala/com/karasiq/shadowcloud/ui/ChallengeHub.scala index bfbc9a6c..2ef1c479 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/ui/ChallengeHub.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/ui/ChallengeHub.scala @@ -41,10 +41,9 @@ class ChallengeHub(implicit as: ActorSystem) { promise.future } - def solve(id: UUID, response: ByteString = ByteString.empty): Unit = challenges.remove(id).foreach { - case (challenge, promise, _) ⇒ - log.info("Challenge solved: {}", challenge.title) - promise.trySuccess(response) + def solve(id: UUID, response: ByteString = ByteString.empty): Unit = challenges.remove(id).foreach { case (challenge, promise, _) ⇒ + log.info("Challenge solved: {}", challenge.title) + promise.trySuccess(response) } override def finalize(): Unit = { diff --git a/core/src/main/scala/com/karasiq/shadowcloud/ui/passwords/console/ConsolePasswordProvider.scala b/core/src/main/scala/com/karasiq/shadowcloud/ui/passwords/console/ConsolePasswordProvider.scala index b2a7a86f..0a143483 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/ui/passwords/console/ConsolePasswordProvider.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/ui/passwords/console/ConsolePasswordProvider.scala @@ -7,7 +7,7 @@ class ConsolePasswordProvider extends PasswordProvider { private[this] lazy val default = new StdInPasswordProvider() def askPassword(id: String): String = System.console() match { - case null => default.askPassword(id) - case console => new String(console.readPassword(s"Enter password ($id): ")) + case null ⇒ default.askPassword(id) + case console ⇒ new String(console.readPassword(s"Enter password ($id): ")) } } diff --git a/core/src/main/scala/com/karasiq/shadowcloud/utils/ChunkUtils.scala b/core/src/main/scala/com/karasiq/shadowcloud/utils/ChunkUtils.scala index d501448f..ead83313 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/utils/ChunkUtils.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/utils/ChunkUtils.scala @@ -38,7 +38,7 @@ private[shadowcloud] object ChunkUtils { } def recoverChunkData(modules: CryptoModuleRegistry, chunk: Chunk): Chunk = { - val withPlain = chunk.copy(data = chunk.data.copy(plain = getPlainBytes(modules, chunk))) + val withPlain = chunk.copy(data = chunk.data.copy(plain = getPlainBytes(modules, chunk))) val withEncrypted = withPlain.copy(data = withPlain.data.copy(encrypted = getEncryptedBytes(modules, withPlain))) withEncrypted } diff --git a/core/src/main/scala/com/karasiq/shadowcloud/utils/SCProviderInstantiator.scala b/core/src/main/scala/com/karasiq/shadowcloud/utils/SCProviderInstantiator.scala index 0a123151..0c78afdd 100644 --- a/core/src/main/scala/com/karasiq/shadowcloud/utils/SCProviderInstantiator.scala +++ b/core/src/main/scala/com/karasiq/shadowcloud/utils/SCProviderInstantiator.scala @@ -18,9 +18,10 @@ private[shadowcloud] final class SCProviderInstantiator(sc: ShadowCloudExtension def getInstance[T](pClass: Class[T]): T = { def tryCreate(argsClasses: Seq[Class[_]], args: Seq[AnyRef]): Option[T] = { try { - Some(pClass.getConstructor(argsClasses:_*).newInstance(args:_*)) - } catch { case _: NoSuchMethodException ⇒ - None + Some(pClass.getConstructor(argsClasses: _*).newInstance(args: _*)) + } catch { + case _: NoSuchMethodException ⇒ + None } } diff --git a/core/src/test/scala/com/karasiq/shadowcloud/test/utils/CoreTestUtils.scala b/core/src/test/scala/com/karasiq/shadowcloud/test/utils/CoreTestUtils.scala index e238145b..9944af3e 100644 --- a/core/src/test/scala/com/karasiq/shadowcloud/test/utils/CoreTestUtils.scala +++ b/core/src/test/scala/com/karasiq/shadowcloud/test/utils/CoreTestUtils.scala @@ -1,6 +1,5 @@ package com.karasiq.shadowcloud.test.utils - import akka.util.ByteString import com.karasiq.shadowcloud.config.{RegionConfig, SCConfig, SerializedProps, StorageConfig} import com.karasiq.shadowcloud.index.diffs.{ChunkIndexDiff, FolderIndexDiff, IndexDiff} @@ -15,9 +14,9 @@ import scala.util.{Random, Try} object CoreTestUtils extends ByteStringImplicits { import TestUtils.{randomBytes, randomString} - val config = SCConfig(ConfigFactory.load().getConfig("shadowcloud")) - val modules = SCModules(config)(new TestProviderInstantiator) - val sha1Hashing = modules.crypto.hashingModule(HashingMethod("SHA1")) + val config = SCConfig(ConfigFactory.load().getConfig("shadowcloud")) + val modules = SCModules(config)(new TestProviderInstantiator) + val sha1Hashing = modules.crypto.hashingModule(HashingMethod("SHA1")) val aesEncryption = modules.crypto.encryptionModule(EncryptionMethod("AES/GCM", 256)) def regionConfig(regionId: RegionId): RegionConfig = { @@ -29,9 +28,9 @@ object CoreTestUtils extends ByteStringImplicits { } def randomChunk: Chunk = { - val data = randomBytes(100) + val data = randomBytes(100) val encParameters = aesEncryption.createParameters() - val encData = aesEncryption.encrypt(data, encParameters) + val encData = aesEncryption.encrypt(data, encParameters) Chunk( Checksum(sha1Hashing.method, sha1Hashing.method, data.length, sha1Hashing.createHash(data), encData.length, sha1Hashing.createHash(encData)), encParameters, @@ -40,18 +39,25 @@ object CoreTestUtils extends ByteStringImplicits { } def randomFile(parent: Path = Path.root): File = { - val chunks = Seq.fill(1)(randomChunk) - val size = chunks.map(_.checksum.size).sum + val chunks = Seq.fill(1)(randomChunk) + val size = chunks.map(_.checksum.size).sum val encSize = chunks.map(_.checksum.encSize).sum - val hash = sha1Hashing.createHash(ByteString.fromChunks(chunks)) + val hash = sha1Hashing.createHash(ByteString.fromChunks(chunks)) val encHash = sha1Hashing.createHash(ByteString.fromEncryptedChunks(chunks)) - File(parent / s"$randomString.txt", FileId.create(), Random.nextInt(10), Timestamp.now, SerializedProps.empty, - Checksum(sha1Hashing.method, sha1Hashing.method, size, hash, encSize, encHash), chunks) + File( + parent / s"$randomString.txt", + FileId.create(), + Random.nextInt(10), + Timestamp.now, + SerializedProps.empty, + Checksum(sha1Hashing.method, sha1Hashing.method, size, hash, encSize, encHash), + chunks + ) } def randomFolder(path: Path = Path.root / randomString): Folder = { val folders = Seq.fill(1)(randomString) - val files = Seq.fill(1)(randomFile(path)) + val files = Seq.fill(1)(randomFile(path)) Folder(path, Timestamp.now, folders.toSet, files.toSet) } diff --git a/core/src/test/scala/com/karasiq/shadowcloud/test/utils/SCExtensionSpec.scala b/core/src/test/scala/com/karasiq/shadowcloud/test/utils/SCExtensionSpec.scala index 44f99513..b9eaaf48 100644 --- a/core/src/test/scala/com/karasiq/shadowcloud/test/utils/SCExtensionSpec.scala +++ b/core/src/test/scala/com/karasiq/shadowcloud/test/utils/SCExtensionSpec.scala @@ -1,7 +1,5 @@ package com.karasiq.shadowcloud.test.utils - - import com.karasiq.shadowcloud.{ShadowCloud, ShadowCloudExtension} abstract class SCExtensionSpec extends ActorSpec with ActorSpecImplicits with ByteStringImplicits { diff --git a/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/BouncyCastleCryptoProvider.scala b/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/BouncyCastleCryptoProvider.scala index e19c2c3a..15f7fa83 100644 --- a/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/BouncyCastleCryptoProvider.scala +++ b/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/BouncyCastleCryptoProvider.scala @@ -1,7 +1,5 @@ package com.karasiq.shadowcloud.crypto.bouncycastle - - import com.karasiq.common.configs.ConfigImplicits import com.karasiq.shadowcloud.crypto.bouncycastle.asymmetric.{ECIESCipherModule, RSACipherModule} import com.karasiq.shadowcloud.crypto.bouncycastle.hashing.{BCDigestModule, BCDigests} @@ -21,9 +19,9 @@ final class BouncyCastleCryptoProvider extends CryptoProvider with ConfigImplici override val encryptionAlgorithms: Set[String] = { BCBlockCiphers.blockAlgorithms ++ - BCBlockCiphers.aeadAlgorithms ++ - BCStreamCiphers.algorithms ++ - Set("ECIES", "RSA") + BCBlockCiphers.aeadAlgorithms ++ + BCStreamCiphers.algorithms ++ + Set("ECIES", "RSA") } // TODO: AESFastEngine, Poly1305 diff --git a/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/asymmetric/BCAsymmetricCipherKeys.scala b/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/asymmetric/BCAsymmetricCipherKeys.scala index c79eef89..9a2b572d 100644 --- a/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/asymmetric/BCAsymmetricCipherKeys.scala +++ b/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/asymmetric/BCAsymmetricCipherKeys.scala @@ -1,7 +1,5 @@ package com.karasiq.shadowcloud.crypto.bouncycastle.asymmetric - - import com.karasiq.shadowcloud.crypto.EncryptionModule import com.karasiq.shadowcloud.crypto.bouncycastle.internal.{BCAsymmetricKeys, KeyUtils} import com.karasiq.shadowcloud.model.crypto.{AsymmetricEncryptionParameters, EncryptionParameters} diff --git a/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/asymmetric/BCAsymmetricCipherModule.scala b/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/asymmetric/BCAsymmetricCipherModule.scala index 205b1b33..b9f9880c 100644 --- a/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/asymmetric/BCAsymmetricCipherModule.scala +++ b/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/asymmetric/BCAsymmetricCipherModule.scala @@ -1,7 +1,5 @@ package com.karasiq.shadowcloud.crypto.bouncycastle.asymmetric - - import akka.util.ByteString import com.karasiq.shadowcloud.crypto.{EncryptionModuleStreamer, OnlyStreamEncryptionModule} import com.karasiq.shadowcloud.utils.ByteStringUnsafe @@ -20,7 +18,8 @@ private[bouncycastle] trait BCAsymmetricBlockCipherStreamer extends EncryptionMo ByteString.fromArrayUnsafe(cipher.processBlock(ByteStringUnsafe.getArray(data), 0, data.length)) } else { // Split to blocks - data.grouped(blockSize) + data + .grouped(blockSize) .map(block ⇒ ByteString.fromArrayUnsafe(cipher.processBlock(block.toArray, 0, block.length))) .fold(ByteString.empty)(_ ++ _) } diff --git a/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/asymmetric/ECIESCipherModule.scala b/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/asymmetric/ECIESCipherModule.scala index e52c9379..a685ee4d 100644 --- a/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/asymmetric/ECIESCipherModule.scala +++ b/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/asymmetric/ECIESCipherModule.scala @@ -26,12 +26,11 @@ import org.bouncycastle.crypto.parsers.ECIESPublicKeyParser private[bouncycastle] object ECIESCipherModule { import ConfigImplicits._ - object defaults { - val digest = HashingMethod("SHA-512") // HashingMethod("SHA3", config = ConfigProps("digest-size" → 512)) - val encodingSize = 32 + val digest = HashingMethod("SHA-512") // HashingMethod("SHA3", config = ConfigProps("digest-size" → 512)) + val encodingSize = 32 val derivationSize = 32 - val macKeySize = 256 + val macKeySize = 256 } def apply(method: EncryptionMethod = EncryptionMethod("ECIES")): ECIESCipherModule = { @@ -57,7 +56,8 @@ private[bouncycastle] object ECIESCipherModule { HexString.encode(ByteString.fromArrayUnsafe(bytes)) } - val newConfig = ConfigProps.toConfig(parameters.method.config) + val newConfig = ConfigProps + .toConfig(parameters.method.config) .withValue("ies.derivation", ConfigValueFactory.fromAnyRef(generateHexString(defaults.derivationSize))) .withValue("ies.encoding", ConfigValueFactory.fromAnyRef(generateHexString(defaults.encodingSize))) @@ -69,9 +69,9 @@ private[bouncycastle] object ECIESCipherModule { method.map(BlockCipherModule.createBlockCipher) } - val agreement = new ECDHBasicAgreement() + val agreement = new ECDHBasicAgreement() val kdfGenerator = new KDF2BytesGenerator(BCDigests.createDigest(options.digestMethod)) - val mac = new HMac(BCDigests.createDigest(options.hmacMethod)) + val mac = new HMac(BCDigests.createDigest(options.hmacMethod)) createBlockCipher(options.blockCipherMethod) match { case Some(blockCipher) ⇒ new IESEngine(agreement, kdfGenerator, mac, blockCipher) @@ -85,11 +85,14 @@ private[bouncycastle] object ECIESCipherModule { //noinspection ConvertExpressionToSAM private def createEphKeyGenerator(method: EncryptionMethod, keyPairGenerator: AsymmetricCipherKeyPairGenerator): EphemeralKeyPairGenerator = { val compressPoints = ConfigProps.toConfig(method.config).withDefault(true, _.getBoolean("compress-ec-points")) - new EphemeralKeyPairGenerator(keyPairGenerator, new KeyEncoder { - def getEncoded(keyParameter: AsymmetricKeyParameter): Array[Byte] = { - keyParameter.asInstanceOf[ECPublicKeyParameters].getQ.getEncoded(compressPoints) + new EphemeralKeyPairGenerator( + keyPairGenerator, + new KeyEncoder { + def getEncoded(keyParameter: AsymmetricKeyParameter): Array[Byte] = { + keyParameter.asInstanceOf[ECPublicKeyParameters].getQ.getEncoded(compressPoints) + } } - }) + ) } private def createPublicKeyParser(method: EncryptionMethod): KeyParser = { @@ -101,7 +104,7 @@ private[bouncycastle] object ECIESCipherModule { private[this] val iesConfig = ConfigProps.toConfig(method.config).getConfigIfExists("ies") val derivation = iesConfig.getHexString("derivation") - val encoding = iesConfig.getHexString("encoding") + val encoding = iesConfig.getHexString("encoding") val macKeySize = iesConfig.withDefault(defaults.macKeySize, _.getInt("mac-key-size")) val digestMethod = getDigestMethod("digest").getOrElse(defaults.digest) @@ -129,8 +132,7 @@ private[bouncycastle] object ECIESCipherModule { } } -private[bouncycastle] final class ECIESCipherModule(val method: EncryptionMethod) - extends BCAsymmetricCipherModule with BCECKeys { +private[bouncycastle] final class ECIESCipherModule(val method: EncryptionMethod) extends BCAsymmetricCipherModule with BCECKeys { private[this] lazy val secureRandom = BCUtils.createSecureRandom() @@ -145,7 +147,7 @@ private[bouncycastle] final class ECIESCipherModule(val method: EncryptionMethod protected class ECIESEngineStreamer extends EncryptionModuleStreamer { private[this] var iesEngine: IESEngine = _ - private[this] val ephKeyGenerator = ECIESCipherModule.createEphKeyGenerator(method, keyPairGenerator) + private[this] val ephKeyGenerator = ECIESCipherModule.createEphKeyGenerator(method, keyPairGenerator) def module: EncryptionModule = { ECIESCipherModule.this @@ -153,7 +155,7 @@ private[bouncycastle] final class ECIESCipherModule(val method: EncryptionMethod def init(encrypt: Boolean, parameters: EncryptionParameters): Unit = { val asymmetricKey = BCAsymmetricCipherKeys.getCipherKey(parameters, encrypt) - val options = IESOptions(parameters.method) + val options = IESOptions(parameters.method) val iesParameters = ECIESCipherModule.toIesParameters(options) iesEngine = ECIESCipherModule.createIesEngine(options) diff --git a/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/asymmetric/RSACipherModule.scala b/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/asymmetric/RSACipherModule.scala index 393c73bd..c98db429 100644 --- a/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/asymmetric/RSACipherModule.scala +++ b/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/asymmetric/RSACipherModule.scala @@ -1,7 +1,5 @@ package com.karasiq.shadowcloud.crypto.bouncycastle.asymmetric - - import com.karasiq.shadowcloud.crypto.bouncycastle.internal.RSAUtils import com.karasiq.shadowcloud.crypto.{EncryptionModule, EncryptionModuleStreamer} import com.karasiq.shadowcloud.model.crypto.{EncryptionMethod, EncryptionParameters} diff --git a/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/hashing/BCDigestModule.scala b/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/hashing/BCDigestModule.scala index 0b326248..d052ab34 100644 --- a/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/hashing/BCDigestModule.scala +++ b/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/hashing/BCDigestModule.scala @@ -43,4 +43,4 @@ private[bouncycastle] final class BCDigestModule(val method: HashingMethod) exte digest.reset() } } -} \ No newline at end of file +} diff --git a/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/hashing/BCDigests.scala b/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/hashing/BCDigests.scala index 6974322e..16564e12 100644 --- a/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/hashing/BCDigests.scala +++ b/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/hashing/BCDigests.scala @@ -9,14 +9,35 @@ import com.karasiq.shadowcloud.model.crypto.HashingMethod import org.bouncycastle.crypto.Digest private[bouncycastle] object BCDigests extends ConfigImplicits { + /** [[org.bouncycastle.jce.provider.BouncyCastleProvider.DIGESTS]] */ val algorithms: Set[String] = Set( - "GOST3411", "Keccak", "MD2", "MD4", "MD5", "SHA1", "RIPEMD128", "RIPEMD160", "RIPEMD256", "RIPEMD320", "SHA224", - "SHA256", "SHA384", "SHA512", "SHA3", "Skein", "SM3", "Tiger", "Whirlpool", "Blake2b" + "GOST3411", + "Keccak", + "MD2", + "MD4", + "MD5", + "SHA1", + "RIPEMD128", + "RIPEMD160", + "RIPEMD256", + "RIPEMD320", + "SHA224", + "SHA256", + "SHA384", + "SHA512", + "SHA3", + "Skein", + "SM3", + "Tiger", + "Whirlpool", + "Blake2b" ) private[this] val digestsWithSize: Set[String] = Set( - "Keccak", "SHA3", "Blake2b" + "Keccak", + "SHA3", + "Blake2b" ) private[this] val digestsWithTwoSizes: Set[String] = Set( @@ -56,14 +77,14 @@ private[bouncycastle] object BCDigests extends ConfigImplicits { } private[this] def createMDWithSize(algorithm: String, method: HashingMethod, defaultSize: Int = 256): MessageDigest = { - val config = ConfigProps.toConfig(method.config) + val config = ConfigProps.toConfig(method.config) val digestSize = config.withDefault(defaultSize, _.getInt("digest-size")) getMDInstance(s"$algorithm-$digestSize") } private[this] def createMDWithTwoSizes(alg: String, method: HashingMethod, defaultStateSize: Int = 256): MessageDigest = { - val config = ConfigProps.toConfig(method.config) - val stateSize = config.withDefault(defaultStateSize, _.getInt("state-size")) + val config = ConfigProps.toConfig(method.config) + val stateSize = config.withDefault(defaultStateSize, _.getInt("state-size")) val digestSize = config.withDefault(stateSize, _.getInt("digest-size")) getMDInstance(s"$alg-$stateSize-$digestSize") } diff --git a/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/hashing/Blake2b.scala b/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/hashing/Blake2b.scala index 3d28bb96..4519fd07 100644 --- a/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/hashing/Blake2b.scala +++ b/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/hashing/Blake2b.scala @@ -9,10 +9,10 @@ import com.karasiq.shadowcloud.model.crypto.HashingMethod private[bouncycastle] object Blake2b { private case class Blake2bOptions(method: HashingMethod) { import com.karasiq.common.configs.ConfigImplicits._ - private[this] val config = ConfigProps.toConfig(method.config) - val digestSize = config.withDefault(256, _.getInt("digest-size")) - val digestKey = config.withDefault(ByteString.empty, _.getHexString("digest-key")) - val digestSalt = config.withDefault(ByteString.empty, _.getHexString("digest-salt")) + private[this] val config = ConfigProps.toConfig(method.config) + val digestSize = config.withDefault(256, _.getInt("digest-size")) + val digestKey = config.withDefault(ByteString.empty, _.getHexString("digest-key")) + val digestSalt = config.withDefault(ByteString.empty, _.getHexString("digest-salt")) val digestPersonalization = config.withDefault(ByteString.empty, _.getHexString("digest-personalization")) } @@ -22,7 +22,7 @@ private[bouncycastle] object Blake2b { if (options.digestKey.nonEmpty) options.digestKey.toArray else null, options.digestSize / 8, if (options.digestSalt.nonEmpty) options.digestSalt.toArray else null, - if (options.digestPersonalization.nonEmpty) options.digestPersonalization.toArray else null, + if (options.digestPersonalization.nonEmpty) options.digestPersonalization.toArray else null ) } } diff --git a/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/internal/BCSymmetricKeys.scala b/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/internal/BCSymmetricKeys.scala index f27be7b6..11d3c2dc 100644 --- a/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/internal/BCSymmetricKeys.scala +++ b/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/internal/BCSymmetricKeys.scala @@ -1,7 +1,5 @@ package com.karasiq.shadowcloud.crypto.bouncycastle.internal - - import akka.util.ByteString import com.karasiq.shadowcloud.crypto.EncryptionModule import com.karasiq.shadowcloud.model.crypto.{EncryptionMethod, EncryptionParameters, SymmetricEncryptionParameters} diff --git a/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/internal/ECUtils.scala b/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/internal/ECUtils.scala index 24126571..7adb5d29 100644 --- a/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/internal/ECUtils.scala +++ b/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/internal/ECUtils.scala @@ -1,6 +1,5 @@ package com.karasiq.shadowcloud.crypto.bouncycastle.internal - import com.karasiq.shadowcloud.config.ConfigProps import com.karasiq.shadowcloud.model.crypto.{CryptoMethod, EncryptionMethod, SignMethod} import org.bouncycastle.crypto.AsymmetricCipherKeyPairGenerator @@ -34,12 +33,14 @@ private[bouncycastle] object ECUtils { getCurveForSize(keySize) } - val curveName = try { - val config = ConfigProps.toConfig(method.config) - Some(config.getString("curve")) - } catch { case NonFatal(_) ⇒ - None - } + val curveName = + try { + val config = ConfigProps.toConfig(method.config) + Some(config.getString("curve")) + } catch { + case NonFatal(_) ⇒ + None + } curveName.getOrElse(getDefaultCurve(method)) } @@ -61,9 +62,9 @@ private[bouncycastle] object ECUtils { } def createKeyGenerator(method: CryptoMethod): AsymmetricCipherKeyPairGenerator = { - val generator = new ECKeyPairGenerator + val generator = new ECKeyPairGenerator val domainParameters = ECUtils.getCurveDomainParameters(method) - val secureRandom = BCUtils.createSecureRandom() + val secureRandom = BCUtils.createSecureRandom() generator.init(new ECKeyGenerationParameters(domainParameters, secureRandom)) generator } diff --git a/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/internal/RSAUtils.scala b/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/internal/RSAUtils.scala index 68687ee3..fc8e5d46 100644 --- a/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/internal/RSAUtils.scala +++ b/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/internal/RSAUtils.scala @@ -10,18 +10,17 @@ import com.karasiq.shadowcloud.model.crypto.{CryptoMethod, EncryptionMethod, Sig private[bouncycastle] object RSAUtils { type KeyGenerator = org.bouncycastle.crypto.AsymmetricCipherKeyPairGenerator - /** - * @see [[org.bouncycastle.jcajce.provider.asymmetric.rsa.KeyPairGeneratorSpi]] + /** @see [[org.bouncycastle.jcajce.provider.asymmetric.rsa.KeyPairGeneratorSpi]] */ object defaults { - val keySize = 2048 - val publicExponent = BigInt(0x10001) + val keySize = 2048 + val publicExponent = BigInt(0x10001) def certainty(keySize: Int): Int = PrimeCertaintyCalculator.getDefaultCertainty(keySize) } private[this] def createKeyGenerator(keySize: Int, publicExponent: BigInt, certainty: Int): KeyGenerator = { - val generator = new RSAKeyPairGenerator - val secureRandom = BCUtils.createSecureRandom() + val generator = new RSAKeyPairGenerator + val secureRandom = BCUtils.createSecureRandom() val rsaParameters = new RSAKeyGenerationParameters(publicExponent.bigInteger, secureRandom, keySize, certainty) generator.init(rsaParameters) generator @@ -38,11 +37,11 @@ private[bouncycastle] object RSAUtils { private[this] val rsaConfig = ConfigProps.toConfig(method.config).getConfigIfExists("rsa") val keySize = method match { - case sm: SignMethod ⇒ sm.keySize + case sm: SignMethod ⇒ sm.keySize case em: EncryptionMethod ⇒ em.keySize - case _ ⇒ defaults.keySize + case _ ⇒ defaults.keySize } val publicExponent = rsaConfig.withDefault(defaults.publicExponent, cfg ⇒ BigInt(cfg.getString("public-exponent"))) - val certainty = rsaConfig.withDefault(defaults.certainty(keySize), _.getInt("certainty")) + val certainty = rsaConfig.withDefault(defaults.certainty(keySize), _.getInt("certainty")) } } diff --git a/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/sign/BCSignKeys.scala b/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/sign/BCSignKeys.scala index 07e39dde..a8dc56f1 100644 --- a/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/sign/BCSignKeys.scala +++ b/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/sign/BCSignKeys.scala @@ -21,4 +21,4 @@ private[bouncycastle] trait BCSignKeys extends BCAsymmetricKeys { self: SignModu val keyPair = keyPairGenerator.generateKeyPair() SignParameters(method, KeyUtils.encodePublicKey(keyPair.getPublic), KeyUtils.encodePrivateKey(keyPair.getPrivate)) } -} \ No newline at end of file +} diff --git a/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/sign/BCSignModule.scala b/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/sign/BCSignModule.scala index e8aa6bba..3787df13 100644 --- a/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/sign/BCSignModule.scala +++ b/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/sign/BCSignModule.scala @@ -50,4 +50,4 @@ private[bouncycastle] trait BCDSAStreamer extends BCSignerStreamer { this.signer = new DSADigestSigner(dsaSigner, BCDigests.createDigest(parameters.method.hashingMethod)) super.init(sign, parameters) } -} \ No newline at end of file +} diff --git a/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/sign/ECDSASignModule.scala b/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/sign/ECDSASignModule.scala index 98d86a0c..fe7ca1dc 100644 --- a/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/sign/ECDSASignModule.scala +++ b/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/sign/ECDSASignModule.scala @@ -1,7 +1,5 @@ package com.karasiq.shadowcloud.crypto.bouncycastle.sign - - import com.karasiq.shadowcloud.crypto.{SignModule, SignModuleStreamer} import com.karasiq.shadowcloud.model.crypto.{HashingMethod, SignMethod} import org.bouncycastle.crypto.DSA @@ -20,6 +18,6 @@ private[bouncycastle] final class ECDSASignModule(val method: SignMethod) extend protected class ECDSASignerStreamer extends BCDSAStreamer { protected val dsaSigner: DSA = new ECDSASigner() - def module: SignModule = ECDSASignModule.this + def module: SignModule = ECDSASignModule.this } } diff --git a/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/sign/RSASignModule.scala b/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/sign/RSASignModule.scala index 64c4448a..5884da3a 100644 --- a/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/sign/RSASignModule.scala +++ b/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/sign/RSASignModule.scala @@ -1,7 +1,5 @@ package com.karasiq.shadowcloud.crypto.bouncycastle.sign - - import com.karasiq.shadowcloud.crypto._ import com.karasiq.shadowcloud.crypto.bouncycastle.hashing.BCDigests import com.karasiq.shadowcloud.crypto.bouncycastle.internal.RSAUtils diff --git a/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/symmetric/AEADBlockCipherModule.scala b/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/symmetric/AEADBlockCipherModule.scala index 041c6ff6..197fddc7 100644 --- a/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/symmetric/AEADBlockCipherModule.scala +++ b/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/symmetric/AEADBlockCipherModule.scala @@ -1,7 +1,5 @@ package com.karasiq.shadowcloud.crypto.bouncycastle.symmetric - - import akka.util.ByteString import com.karasiq.common.configs.ConfigImplicits import com.karasiq.shadowcloud.config.ConfigProps @@ -39,10 +37,10 @@ private[bouncycastle] object AEADBlockCipherModule { private case class AEADCipherOptions(method: EncryptionMethod) { import ConfigImplicits._ private[this] val config = ConfigProps.toConfig(method.config) - val customBlockSize = config.optional(_.getInt("block-size")) - val nonceSize = config.withDefault(BCBlockCiphers.getNonceSize(method.algorithm, customBlockSize), _.getInt("nonce-size")) - val additionalDataSize = config.withDefault(0, _.getInt("ad-size")) - val macSize = config.withDefault(BCBlockCiphers.getAeadMacSize(method.algorithm, customBlockSize), _.getInt("mac-size")) + val customBlockSize = config.optional(_.getInt("block-size")) + val nonceSize = config.withDefault(BCBlockCiphers.getNonceSize(method.algorithm, customBlockSize), _.getInt("nonce-size")) + val additionalDataSize = config.withDefault(0, _.getInt("ad-size")) + val macSize = config.withDefault(BCBlockCiphers.getAeadMacSize(method.algorithm, customBlockSize), _.getInt("mac-size")) } private def toAEADParameters(parameters: EncryptionParameters): AEADParameters = { @@ -51,12 +49,13 @@ private[bouncycastle] object AEADBlockCipherModule { @inline def splitNonce(value: ByteString): (ByteString, ByteString) = value.splitAt(options.nonceSize) - val symmetricParameters = EncryptionParameters.symmetric(parameters) + val symmetricParameters = EncryptionParameters.symmetric(parameters) val (nonce, additionalData) = splitNonce(symmetricParameters.nonce) new AEADParameters( new KeyParameter(symmetricParameters.key.toArrayUnsafe), - options.macSize, nonce.toArrayUnsafe, + options.macSize, + nonce.toArrayUnsafe, if (additionalData.nonEmpty) additionalData.toArrayUnsafe else null ) } @@ -67,8 +66,7 @@ private[bouncycastle] object AEADBlockCipherModule { } } -private[bouncycastle] class AEADBlockCipherModule(defaultOptions: AEADCipherOptions) - extends OnlyStreamEncryptionModule with BCSymmetricKeys { +private[bouncycastle] class AEADBlockCipherModule(defaultOptions: AEADCipherOptions) extends OnlyStreamEncryptionModule with BCSymmetricKeys { val method: EncryptionMethod = defaultOptions.method protected val nonceSize: Int = defaultOptions.nonceSize + defaultOptions.additionalDataSize diff --git a/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/symmetric/BCBlockCiphers.scala b/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/symmetric/BCBlockCiphers.scala index 8924a899..0cd50f24 100644 --- a/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/symmetric/BCBlockCiphers.scala +++ b/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/symmetric/BCBlockCiphers.scala @@ -21,20 +21,21 @@ private[bouncycastle] object BCBlockCiphers { private[this] val blockCipherSpecs = { def baseCipher(_algorithm: String, _createInstance: ⇒ BlockCipher, _keySize: Int, _blockSize: Int): BlockCipherSpec = new BlockCipherSpec { - def algorithm: String = _algorithm - def createInstance(): BlockCipher = _createInstance + def algorithm: String = _algorithm + def createInstance(): BlockCipher = _createInstance def createInstanceWithBlockSize(blockSize: Int): BlockCipher = throw new NotImplementedError() - def keySize: Int = _keySize - def blockSize: Int = _blockSize + def keySize: Int = _keySize + def blockSize: Int = _blockSize } - def tweakableCipher(_algorithm: String, _createInstance: Int ⇒ BlockCipher, _keySize: Int, _blockSize: Int): BlockCipherSpec = new BlockCipherSpec { - def algorithm: String = _algorithm - def createInstance(): BlockCipher = createInstanceWithBlockSize(this.blockSize) - def createInstanceWithBlockSize(blockSize: Int): BlockCipher = _createInstance(blockSize) - def keySize: Int = _keySize - def blockSize: Int = _blockSize - } + def tweakableCipher(_algorithm: String, _createInstance: Int ⇒ BlockCipher, _keySize: Int, _blockSize: Int): BlockCipherSpec = + new BlockCipherSpec { + def algorithm: String = _algorithm + def createInstance(): BlockCipher = createInstanceWithBlockSize(this.blockSize) + def createInstanceWithBlockSize(blockSize: Int): BlockCipher = _createInstance(blockSize) + def keySize: Int = _keySize + def blockSize: Int = _blockSize + } // https://www.bouncycastle.org/specifications.html // (\w+)Engine\t(?:[0 ,..]+)?([\d]+)(?:[\d ,..]+)?\t(\d+)(?:.*) => baseCipher("$1", new $1Engine, $2, $3), @@ -74,7 +75,7 @@ private[bouncycastle] object BCBlockCiphers { val algorithms: Set[String] = blockCipherSpecs.keySet val blockModes: Set[String] = Set("CBC", "CFB", "OFB", "CTR") - val aeadModes: Set[String] = Set("GCM", "CCM", "EAX", "OCB") + val aeadModes: Set[String] = Set("GCM", "CCM", "EAX", "OCB") val blockAlgorithms: Set[String] = for (alg ← algorithms; mode ← blockModes) @@ -92,8 +93,7 @@ private[bouncycastle] object BCBlockCiphers { aeadAlgorithms.contains(algorithm) } - /** - * @see [[org.bouncycastle.jce.provider.BouncyCastleProvider#SYMMETRIC_CIPHERS]] + /** @see [[org.bouncycastle.jce.provider.BouncyCastleProvider#SYMMETRIC_CIPHERS]] * [[https://www.bouncycastle.org/specifications.html]] */ def createBaseEngine(algorithm: String, blockSize: Option[Int] = None): BlockCipher = { @@ -141,7 +141,7 @@ private[bouncycastle] object BCBlockCiphers { def getAeadMacSize(algorithm: String, blockSize: Option[Int] = None): Int = { val (baseAlgorithm, mode) = BCUtils.algorithmAndMode(algorithm) - val cipherBlockBits = blockSize.getOrElse(getBlockSize(baseAlgorithm)) + val cipherBlockBits = blockSize.getOrElse(getBlockSize(baseAlgorithm)) mode match { case "CCM" | "EAX" ⇒ cipherBlockBits / 2 diff --git a/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/symmetric/BlockCipherModule.scala b/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/symmetric/BlockCipherModule.scala index cab8d141..bdc5890f 100644 --- a/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/symmetric/BlockCipherModule.scala +++ b/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/symmetric/BlockCipherModule.scala @@ -1,7 +1,5 @@ package com.karasiq.shadowcloud.crypto.bouncycastle.symmetric - - import akka.util.ByteString import com.karasiq.common.configs.ConfigImplicits import com.karasiq.shadowcloud.config.ConfigProps @@ -37,19 +35,18 @@ private[bouncycastle] object BlockCipherModule { private case class BlockCipherOptions(method: EncryptionMethod) { import ConfigImplicits._ private[this] val config = ConfigProps.toConfig(method.config) - val customBlockSize = config.optional(_.getInt("block-size")) - val nonceSize = config.withDefault(BCBlockCiphers.getNonceSize(method.algorithm, customBlockSize), _.getInt("nonce-size")) + val customBlockSize = config.optional(_.getInt("block-size")) + val nonceSize = config.withDefault(BCBlockCiphers.getNonceSize(method.algorithm, customBlockSize), _.getInt("nonce-size")) } private[bouncycastle] def createBlockCipher(method: EncryptionMethod): PaddedBufferedBlockCipher = { - val options = BlockCipherOptions(method) + val options = BlockCipherOptions(method) val baseCipher = BCBlockCiphers.createBlockCipher(method.algorithm, options.customBlockSize) BCBlockCiphers.toPaddedBufferedBlockCipher(baseCipher) } } -private[bouncycastle] class BlockCipherModule(defaultOptions: BlockCipherOptions) - extends OnlyStreamEncryptionModule with BCSymmetricKeys { +private[bouncycastle] class BlockCipherModule(defaultOptions: BlockCipherOptions) extends OnlyStreamEncryptionModule with BCSymmetricKeys { val method: EncryptionMethod = defaultOptions.method protected val nonceSize: Int = defaultOptions.nonceSize @@ -72,7 +69,7 @@ private[bouncycastle] class BlockCipherModule(defaultOptions: BlockCipherOptions def process(data: ByteString): ByteString = { requireInitialized() - val outArray = new Array[Byte](cipher.getUpdateOutputSize(data.length)) + val outArray = new Array[Byte](cipher.getUpdateOutputSize(data.length)) val outLength = cipher.processBytes(data.toArrayUnsafe, 0, data.length, outArray, 0) if (outArray.length == outLength) ByteString.fromArrayUnsafe(outArray) else ByteString.fromArray(outArray, 0, outLength) @@ -80,7 +77,7 @@ private[bouncycastle] class BlockCipherModule(defaultOptions: BlockCipherOptions def finish(): ByteString = { requireInitialized() - val outArray = new Array[Byte](cipher.getOutputSize(0)) + val outArray = new Array[Byte](cipher.getOutputSize(0)) val outLength = cipher.doFinal(outArray, 0) if (outArray.length == outLength) ByteString.fromArrayUnsafe(outArray) else ByteString.fromArray(outArray, 0, outLength) diff --git a/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/symmetric/StreamCipherModule.scala b/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/symmetric/StreamCipherModule.scala index 4248bb23..ce3e4055 100644 --- a/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/symmetric/StreamCipherModule.scala +++ b/crypto/bouncycastle/src/main/scala/com/karasiq/shadowcloud/crypto/bouncycastle/symmetric/StreamCipherModule.scala @@ -1,7 +1,5 @@ package com.karasiq.shadowcloud.crypto.bouncycastle.symmetric - - import akka.util.ByteString import com.karasiq.shadowcloud.crypto._ import com.karasiq.shadowcloud.crypto.bouncycastle.internal.{BCSymmetricKeys, BCUtils} @@ -28,8 +26,7 @@ private[bouncycastle] object StreamCipherModule { } } -private[bouncycastle] final class StreamCipherModule(val method: EncryptionMethod) - extends OnlyStreamEncryptionModule with BCSymmetricKeys { +private[bouncycastle] final class StreamCipherModule(val method: EncryptionMethod) extends OnlyStreamEncryptionModule with BCSymmetricKeys { protected val nonceSize: Int = BCStreamCiphers.getNonceSize(method.algorithm) @@ -51,7 +48,7 @@ private[bouncycastle] final class StreamCipherModule(val method: EncryptionMetho def process(data: ByteString): ByteString = { require(cipher ne null, "Not initialized") - val outArray = new Array[Byte](data.length) + val outArray = new Array[Byte](data.length) val outLength = cipher.processBytes(data.toArrayUnsafe, 0, data.length, outArray, 0) if (outArray.length == outLength) ByteString.fromArrayUnsafe(outArray) else ByteString.fromArray(outArray, 0, outLength) diff --git a/crypto/bouncycastle/src/test/scala/com/karasiq/shadowcloud/crypto/bouncycastle/test/BouncyCastleTest.scala b/crypto/bouncycastle/src/test/scala/com/karasiq/shadowcloud/crypto/bouncycastle/test/BouncyCastleTest.scala index 0f2ec25a..180cf7c6 100644 --- a/crypto/bouncycastle/src/test/scala/com/karasiq/shadowcloud/crypto/bouncycastle/test/BouncyCastleTest.scala +++ b/crypto/bouncycastle/src/test/scala/com/karasiq/shadowcloud/crypto/bouncycastle/test/BouncyCastleTest.scala @@ -18,17 +18,24 @@ import org.scalatest.{FlatSpec, Matchers} //noinspection RedundantDefaultArgument class BouncyCastleTest extends FlatSpec with Matchers { val testVectors = CryptoTestVectors("bouncycastle") - val provider = new BouncyCastleCryptoProvider - val testData = ByteString("# First, make a nonce: A single-use value never repeated under the same key\n# The nonce isn't secret, and can be sent with the ciphertext.\n# The cipher instance has a nonce_bytes method for determining how many bytes should be in a nonce") + val provider = new BouncyCastleCryptoProvider + val testData = ByteString( + "# First, make a nonce: A single-use value never repeated under the same key\n# The nonce isn't secret, and can be sent with the ciphertext.\n# The cipher instance has a nonce_bytes method for determining how many bytes should be in a nonce" + ) // ----------------------------------------------------------------------- // Encryption // ----------------------------------------------------------------------- testSymmetricEncryption("AES/GCM (AD tag)", AEADBlockCipherModule(EncryptionMethod("AES/GCM", config = ConfigProps("ad-size" → 16))), 32, 12 + 16) - testSymmetricEncryption("Threefish (1024)", BlockCipherModule(EncryptionMethod("Threefish/CBC", 1024, config = ConfigProps("block-size" → 1024))), 128, 128) + testSymmetricEncryption( + "Threefish (1024)", + BlockCipherModule(EncryptionMethod("Threefish/CBC", 1024, config = ConfigProps("block-size" → 1024))), + 128, + 128 + ) BCBlockCiphers.blockAlgorithms.foreach { algorithm ⇒ - val keySize = BCBlockCiphers.getKeySize(algorithm) + val keySize = BCBlockCiphers.getKeySize(algorithm) val nonceSize = BCBlockCiphers.getNonceSize(algorithm) testSymmetricEncryption(algorithm, BlockCipherModule(EncryptionMethod(algorithm, keySize)), keySize / 8, nonceSize) } @@ -38,9 +45,10 @@ class BouncyCastleTest extends FlatSpec with Matchers { // val blockSize = BCBlockCiphers.getBlockSize(algorithm) val nonceSize = BCBlockCiphers.getNonceSize(algorithm) - /* if (blockSize == 128) */ testSymmetricEncryption(algorithm, AEADBlockCipherModule(EncryptionMethod(algorithm, keySize)), keySize / 8, nonceSize) + /* if (blockSize == 128) */ + testSymmetricEncryption(algorithm, AEADBlockCipherModule(EncryptionMethod(algorithm, keySize)), keySize / 8, nonceSize) } - + testSymmetricEncryption("Salsa20", StreamCipherModule.Salsa20(), 32, 8) testSymmetricEncryption("XSalsa20", StreamCipherModule.XSalsa20(), 32, 24) testSymmetricEncryption("ChaCha20", StreamCipherModule.ChaCha20(), 32, 8) @@ -59,8 +67,26 @@ class BouncyCastleTest extends FlatSpec with Matchers { // Hashes // ----------------------------------------------------------------------- val testAlgorithms = Seq( - "GOST3411", "Keccak", "MD2", "MD4", "MD5", "SHA1", "RIPEMD128", "RIPEMD160", "RIPEMD256", "RIPEMD320", "SHA224", - "SHA256", "SHA384", "SHA512", "SHA3", "Skein", "SM3", "Tiger", "Whirlpool", "Blake2b" + "GOST3411", + "Keccak", + "MD2", + "MD4", + "MD5", + "SHA1", + "RIPEMD128", + "RIPEMD160", + "RIPEMD256", + "RIPEMD320", + "SHA224", + "SHA256", + "SHA384", + "SHA512", + "SHA3", + "Skein", + "SM3", + "Tiger", + "Whirlpool", + "Blake2b" ) val testHashes = Seq( @@ -94,20 +120,39 @@ class BouncyCastleTest extends FlatSpec with Matchers { testHashing(alg, hash) } - testHashing("Blake2b+key", BCDigestModule(HashingMethod("Blake2b", config = ConfigProps("digest-key" → "824396f4585a22b2c4b36df76f55e669d4edfb423970071b6b616ce454a95400"))), "717fc02f9817eb24cc3f9e803fddebf81fc18b415537b1ea9bb08691215d162d") + testHashing( + "Blake2b+key", + BCDigestModule(HashingMethod("Blake2b", config = ConfigProps("digest-key" → "824396f4585a22b2c4b36df76f55e669d4edfb423970071b6b616ce454a95400"))), + "717fc02f9817eb24cc3f9e803fddebf81fc18b415537b1ea9bb08691215d162d" + ) - testHashing("Blake2b+key+salt+personalization", BCDigestModule(HashingMethod("Blake2b", config = ConfigProps( - "digest-key" → "824396f4585a22b2c4b36df76f55e669d4edfb423970071b6b616ce454a95400", - "digest-salt" → "824396f4585a22b2c4b36df76f55e669", - "digest-personalization" → "d4edfb423970071b6b616ce454a95400" - ))), "0eb3a9e54089752c9972df3bf64cf5df677b58747a8194ab58fd12d516475fc8") + testHashing( + "Blake2b+key+salt+personalization", + BCDigestModule( + HashingMethod( + "Blake2b", + config = ConfigProps( + "digest-key" → "824396f4585a22b2c4b36df76f55e669d4edfb423970071b6b616ce454a95400", + "digest-salt" → "824396f4585a22b2c4b36df76f55e669", + "digest-personalization" → "d4edfb423970071b6b616ce454a95400" + ) + ) + ), + "0eb3a9e54089752c9972df3bf64cf5df677b58747a8194ab58fd12d516475fc8" + ) - testHashing("Blake2b-512", BCDigestModule(HashingMethod("Blake2b", config = ConfigProps("digest-size" → 512))), - "9f84251be0c325ad771696302e9ed3cd174f84ffdd0b8de49664e9a3ea934b89a4d008581cd5803b80b3284116174b3c4a79a5029996eb59edc1fbacfd18204e") + testHashing( + "Blake2b-512", + BCDigestModule(HashingMethod("Blake2b", config = ConfigProps("digest-size" → 512))), + "9f84251be0c325ad771696302e9ed3cd174f84ffdd0b8de49664e9a3ea934b89a4d008581cd5803b80b3284116174b3c4a79a5029996eb59edc1fbacfd18204e" + ) - testHashing("Skein-1024-1024", BCDigestModule(HashingMethod("Skein", config = ConfigProps("state-size" → 1024, "digest-size" → 1024))), + testHashing( + "Skein-1024-1024", + BCDigestModule(HashingMethod("Skein", config = ConfigProps("state-size" → 1024, "digest-size" → 1024))), "66588dbe2beb3b9cea762f42e3abaa9dc406bfa005fed3579089d8d2c5807453aa6cb0f8e69134ad47405c843e9a08c51da931827957f06ca58b3e8fe658993e" + - "1ca87d19a09bc168cc5845bc3235050f8dd59c8f8ec302bbdff4508b16c1c7cef694e1a4c84c250132d445637e0a84772196162a5815c38e45ff3dac4374f567") + "1ca87d19a09bc168cc5845bc3235050f8dd59c8f8ec302bbdff4508b16c1c7cef694e1a4c84c250132d445637e0a84772196162a5815c38e45ff3dac4374f567" + ) // ----------------------------------------------------------------------- // Signatures @@ -130,7 +175,7 @@ class BouncyCastleTest extends FlatSpec with Matchers { it should "encrypt data" in { val parameters = module.createParameters() - val encrypted = module.encrypt(testData, parameters) + val encrypted = module.encrypt(testData, parameters) encrypted.length should be >= testData.length encrypted should not be testData val decrypted = module.decrypt(encrypted, parameters) @@ -141,7 +186,7 @@ class BouncyCastleTest extends FlatSpec with Matchers { private[this] def testAsymmetricEncryption(name: String, module: EncryptionModule): Unit = { s"$name module" should "generate key" in { - val parameters = EncryptionParameters.asymmetric(module.createParameters()) + val parameters = EncryptionParameters.asymmetric(module.createParameters()) val parameters1 = EncryptionParameters.asymmetric(module.updateParameters(parameters)) parameters1 shouldBe parameters println(parameters) @@ -149,7 +194,7 @@ class BouncyCastleTest extends FlatSpec with Matchers { it should "encrypt data" in { val parameters = module.createParameters() - val encrypted = module.encrypt(testData, parameters) + val encrypted = module.encrypt(testData, parameters) encrypted.length should be >= testData.length val decrypted = module.decrypt(encrypted, parameters) decrypted shouldBe testData @@ -175,8 +220,9 @@ class BouncyCastleTest extends FlatSpec with Matchers { try { val module = provider.hashing(HashingMethod(name)) testHashing(module.method.algorithm, module, testHash) - } catch { case _: NoSuchAlgorithmException ⇒ - println(s"No such algorithm: $name") + } catch { + case _: NoSuchAlgorithmException ⇒ + println(s"No such algorithm: $name") } } @@ -189,7 +235,7 @@ class BouncyCastleTest extends FlatSpec with Matchers { it should "sign data" in { val parameters = module.createParameters() - val signature = module.sign(testData, parameters) + val signature = module.sign(testData, parameters) module.verify(testData, signature, parameters) shouldBe true module.verify(testData, signature.reverse, parameters) shouldBe false } diff --git a/crypto/libsodium/src/main/scala/com/karasiq/shadowcloud/crypto/libsodium/LibSodiumCryptoProvider.scala b/crypto/libsodium/src/main/scala/com/karasiq/shadowcloud/crypto/libsodium/LibSodiumCryptoProvider.scala index cdad93a6..69f66996 100644 --- a/crypto/libsodium/src/main/scala/com/karasiq/shadowcloud/crypto/libsodium/LibSodiumCryptoProvider.scala +++ b/crypto/libsodium/src/main/scala/com/karasiq/shadowcloud/crypto/libsodium/LibSodiumCryptoProvider.scala @@ -1,7 +1,5 @@ package com.karasiq.shadowcloud.crypto.libsodium - - import com.karasiq.shadowcloud.crypto.libsodium.asymmetric.SealedBoxModule import com.karasiq.shadowcloud.crypto.libsodium.hashing.{Blake2bModule, MultiPartHashModule} import com.karasiq.shadowcloud.crypto.libsodium.internal.LSUtils @@ -11,7 +9,7 @@ import com.karasiq.shadowcloud.providers.CryptoProvider final class LibSodiumCryptoProvider extends CryptoProvider { private[this] val libraryLoaded = LSUtils.isLibraryAvailable - private[this] val aesAvailable = LSUtils.isAesAvailable + private[this] val aesAvailable = LSUtils.isAesAvailable override val hashingAlgorithms: Set[String] = ifLoaded(super.hashingAlgorithms) { Set("SHA256", "SHA512", "Blake2b") diff --git a/crypto/libsodium/src/main/scala/com/karasiq/shadowcloud/crypto/libsodium/asymmetric/SealedBoxModule.scala b/crypto/libsodium/src/main/scala/com/karasiq/shadowcloud/crypto/libsodium/asymmetric/SealedBoxModule.scala index 9db650d8..ac0afa01 100644 --- a/crypto/libsodium/src/main/scala/com/karasiq/shadowcloud/crypto/libsodium/asymmetric/SealedBoxModule.scala +++ b/crypto/libsodium/src/main/scala/com/karasiq/shadowcloud/crypto/libsodium/asymmetric/SealedBoxModule.scala @@ -13,11 +13,10 @@ private[libsodium] object SealedBoxModule { def apply(method: EncryptionMethod = EncryptionMethod(Algorithm)): SealedBoxModule = { new SealedBoxModule(method) - } + } } -/** - * @see [[https://download.libsodium.org/doc/public-key_cryptography/sealed_boxes.html]] +/** @see [[https://download.libsodium.org/doc/public-key_cryptography/sealed_boxes.html]] */ private[libsodium] final class SealedBoxModule(val method: EncryptionMethod) extends EncryptionModule { def createParameters(): EncryptionParameters = { @@ -35,15 +34,15 @@ private[libsodium] final class SealedBoxModule(val method: EncryptionMethod) ext def encrypt(data: ByteString, parameters: EncryptionParameters) = { val asymmetricParameters = EncryptionParameters.asymmetric(parameters) - val sealedBox = new SealedBox(asymmetricParameters.publicKey.toArrayUnsafe) - val outArray = sealedBox.encrypt(data.toArrayUnsafe) + val sealedBox = new SealedBox(asymmetricParameters.publicKey.toArrayUnsafe) + val outArray = sealedBox.encrypt(data.toArrayUnsafe) ByteString.fromArrayUnsafe(outArray) } def decrypt(data: ByteString, parameters: EncryptionParameters) = { val asymmetricParameters = EncryptionParameters.asymmetric(parameters) - val sealedBox = new SealedBox(asymmetricParameters.publicKey.toArrayUnsafe, asymmetricParameters.privateKey.toArrayUnsafe) - val outArray = sealedBox.decrypt(data.toArrayUnsafe) + val sealedBox = new SealedBox(asymmetricParameters.publicKey.toArrayUnsafe, asymmetricParameters.privateKey.toArrayUnsafe) + val outArray = sealedBox.decrypt(data.toArrayUnsafe) ByteString.fromArrayUnsafe(outArray) } } diff --git a/crypto/libsodium/src/main/scala/com/karasiq/shadowcloud/crypto/libsodium/hashing/Blake2bModule.scala b/crypto/libsodium/src/main/scala/com/karasiq/shadowcloud/crypto/libsodium/hashing/Blake2bModule.scala index d052ca8a..b5d200c3 100644 --- a/crypto/libsodium/src/main/scala/com/karasiq/shadowcloud/crypto/libsodium/hashing/Blake2bModule.scala +++ b/crypto/libsodium/src/main/scala/com/karasiq/shadowcloud/crypto/libsodium/hashing/Blake2bModule.scala @@ -22,18 +22,21 @@ private[libsodium] object Blake2bModule { private case class DigestOptions(method: HashingMethod) { import ConfigImplicits._ private[this] val config = ConfigProps.toConfig(method.config) - val digestSize = config.withDefault(256, _.getInt("digest-size")) - val digestKey = config.withDefault(ByteString.empty, _.getHexString("digest-key")) + val digestSize = config.withDefault(256, _.getInt("digest-size")) + val digestKey = config.withDefault(ByteString.empty, _.getHexString("digest-key")) - require(digestKey.isEmpty || (digestKey.length >= NaCl.Sodium.CRYPTO_GENERICHASH_KEYBYTES_MIN && - digestKey.length <= NaCl.Sodium.CRYPTO_GENERICHASH_KEYBYTES_MAX), "Invalid digest key") + require( + digestKey.isEmpty || (digestKey.length >= NaCl.Sodium.CRYPTO_GENERICHASH_KEYBYTES_MIN && + digestKey.length <= NaCl.Sodium.CRYPTO_GENERICHASH_KEYBYTES_MAX), + "Invalid digest key" + ) } } private[libsodium] final class Blake2bModule(options: DigestOptions) extends StreamHashingModule { - val method: HashingMethod = options.method + val method: HashingMethod = options.method private[this] val outBytes = options.digestSize / 8 - private[this] val sodium = NaCl.sodium() + private[this] val sodium = NaCl.sodium() def createStreamer(): HashingModuleStreamer = { new Blake2bStreamer @@ -42,9 +45,12 @@ private[libsodium] final class Blake2bModule(options: DigestOptions) extends Str def createHash(data: ByteString): ByteString = { val outArray = new Array[Byte](outBytes) sodium.crypto_generichash( - outArray, outArray.length, - ByteStringUnsafe.getArray(data), data.length, - ByteStringUnsafe.getArray(options.digestKey), options.digestKey.length + outArray, + outArray.length, + ByteStringUnsafe.getArray(data), + data.length, + ByteStringUnsafe.getArray(options.digestKey), + options.digestKey.length ) ByteString.fromArrayUnsafe(outArray) } diff --git a/crypto/libsodium/src/main/scala/com/karasiq/shadowcloud/crypto/libsodium/hashing/MultiPartHashModule.scala b/crypto/libsodium/src/main/scala/com/karasiq/shadowcloud/crypto/libsodium/hashing/MultiPartHashModule.scala index 6a12ce71..691df769 100644 --- a/crypto/libsodium/src/main/scala/com/karasiq/shadowcloud/crypto/libsodium/hashing/MultiPartHashModule.scala +++ b/crypto/libsodium/src/main/scala/com/karasiq/shadowcloud/crypto/libsodium/hashing/MultiPartHashModule.scala @@ -18,7 +18,7 @@ private[libsodium] object MultiPartHashModule { } private[libsodium] final class MultiPartHashModule(val method: HashingMethod, newInstance: Hash ⇒ Hash.MultiPartHash) - extends OnlyStreamHashingModule { + extends OnlyStreamHashingModule { def createStreamer(): HashingModuleStreamer = { new MultiPartHashStreamer(new Hash(), newInstance) diff --git a/crypto/libsodium/src/main/scala/com/karasiq/shadowcloud/crypto/libsodium/internal/LSUtils.scala b/crypto/libsodium/src/main/scala/com/karasiq/shadowcloud/crypto/libsodium/internal/LSUtils.scala index ee207692..dd84046c 100644 --- a/crypto/libsodium/src/main/scala/com/karasiq/shadowcloud/crypto/libsodium/internal/LSUtils.scala +++ b/crypto/libsodium/src/main/scala/com/karasiq/shadowcloud/crypto/libsodium/internal/LSUtils.scala @@ -1,9 +1,8 @@ package com.karasiq.shadowcloud.crypto.libsodium.internal - import org.abstractj.kalium.NaCl -import scala.util.control.{Exception => ExcControl} +import scala.util.control.{Exception ⇒ ExcControl} private[libsodium] object LSUtils { type LSRandom = org.abstractj.kalium.crypto.Random @@ -15,7 +14,7 @@ private[libsodium] object LSUtils { } lazy val isLibraryAvailable: Boolean = sodiumInstance.isDefined - lazy val isAesAvailable: Boolean = isLibraryAvailable && sodiumInstance.forall(_.crypto_aead_aes256gcm_is_available() == 1) + lazy val isAesAvailable: Boolean = isLibraryAvailable && sodiumInstance.forall(_.crypto_aead_aes256gcm_is_available() == 1) def createSecureRandom(): LSRandom = { new LSRandom diff --git a/crypto/libsodium/src/main/scala/com/karasiq/shadowcloud/crypto/libsodium/signing/CryptoSignModule.scala b/crypto/libsodium/src/main/scala/com/karasiq/shadowcloud/crypto/libsodium/signing/CryptoSignModule.scala index a2491565..668290c7 100644 --- a/crypto/libsodium/src/main/scala/com/karasiq/shadowcloud/crypto/libsodium/signing/CryptoSignModule.scala +++ b/crypto/libsodium/src/main/scala/com/karasiq/shadowcloud/crypto/libsodium/signing/CryptoSignModule.scala @@ -15,19 +15,18 @@ private[libsodium] object CryptoSignModule { } } -/** - * @see [[https://download.libsodium.org/doc/public-key_cryptography/public-key_signatures.html]] +/** @see [[https://download.libsodium.org/doc/public-key_cryptography/public-key_signatures.html]] */ private[libsodium] final class CryptoSignModule(val method: SignMethod) extends SignModule { def createParameters(): SignParameters = { val signingKey = new SigningKey() - val verifyKey = signingKey.getVerifyKey + val verifyKey = signingKey.getVerifyKey SignParameters(method, publicKey = ByteString.fromArrayUnsafe(verifyKey.toBytes), privateKey = ByteString.fromArrayUnsafe(signingKey.toBytes)) } def sign(data: ByteString, parameters: SignParameters): ByteString = { val signingKey = new SigningKey(parameters.privateKey.toArrayUnsafe) - val signature = signingKey.sign(data.toArrayUnsafe) + val signature = signingKey.sign(data.toArrayUnsafe) ByteString.fromArrayUnsafe(signature) } @@ -35,8 +34,9 @@ private[libsodium] final class CryptoSignModule(val method: SignMethod) extends val verifyKey = new VerifyKey(parameters.publicKey.toArrayUnsafe) try { verifyKey.verify(data.toArrayUnsafe, signature.toArrayUnsafe) - } catch { case exc: RuntimeException if exc.getMessage.contains("signature was forged or corrupted") ⇒ - false + } catch { + case exc: RuntimeException if exc.getMessage.contains("signature was forged or corrupted") ⇒ + false } } } diff --git a/crypto/libsodium/src/main/scala/com/karasiq/shadowcloud/crypto/libsodium/symmetric/AEADCipherModule.scala b/crypto/libsodium/src/main/scala/com/karasiq/shadowcloud/crypto/libsodium/symmetric/AEADCipherModule.scala index 8f2480b7..4704dc52 100644 --- a/crypto/libsodium/src/main/scala/com/karasiq/shadowcloud/crypto/libsodium/symmetric/AEADCipherModule.scala +++ b/crypto/libsodium/src/main/scala/com/karasiq/shadowcloud/crypto/libsodium/symmetric/AEADCipherModule.scala @@ -12,10 +12,10 @@ import com.karasiq.shadowcloud.model.crypto.{EncryptionMethod, EncryptionParamet import com.karasiq.shadowcloud.utils.ByteStringUnsafe private[libsodium] object AEADCipherModule extends SymmetricConstants { - val AESKeyBytes: Int = Sodium.CRYPTO_AEAD_AES256GCM_KEYBYTES + val AESKeyBytes: Int = Sodium.CRYPTO_AEAD_AES256GCM_KEYBYTES val AESNonceBytes: Int = Sodium.CRYPTO_AEAD_AES256GCM_NPUBBYTES - val KeyBytes: Int = Sodium.CRYPTO_AEAD_CHACHA20POLY1305_KEYBYTES - val NonceBytes: Int = Sodium.CRYPTO_AEAD_CHACHA20POLY1305_NPUBBYTES + val KeyBytes: Int = Sodium.CRYPTO_AEAD_CHACHA20POLY1305_KEYBYTES + val NonceBytes: Int = Sodium.CRYPTO_AEAD_CHACHA20POLY1305_NPUBBYTES def apply(method: EncryptionMethod): AEADCipherModule = { new AEADCipherModule(AEADCipherOptions(method)) @@ -32,22 +32,22 @@ private[libsodium] object AEADCipherModule extends SymmetricConstants { private case class AEADCipherOptions(method: EncryptionMethod) { import ConfigImplicits._ private[this] val config = ConfigProps.toConfig(method.config) - val useAes = method.algorithm == "AES/GCM" - val additionalDataSize = config.withDefault(0, _.getInt("ad-size")) + val useAes = method.algorithm == "AES/GCM" + val additionalDataSize = config.withDefault(0, _.getInt("ad-size")) - val keySize: Int = if (useAes) AESKeyBytes else KeyBytes + val keySize: Int = if (useAes) AESKeyBytes else KeyBytes val nonceSize: Int = if (useAes) AESNonceBytes else NonceBytes } } private[libsodium] final class AEADCipherModule(defaultOptions: AEADCipherOptions) extends SymmetricCipherModule { val method: EncryptionMethod = defaultOptions.method - protected val keySize: Int = defaultOptions.keySize + protected val keySize: Int = defaultOptions.keySize protected val nonceSize: Int = defaultOptions.nonceSize + defaultOptions.additionalDataSize def encrypt(data: ByteString, parameters: EncryptionParameters): ByteString = { val symmetricParameters = EncryptionParameters.symmetric(parameters) - val aeadOptions = AEADCipherOptions(symmetricParameters.method) + val aeadOptions = AEADCipherOptions(symmetricParameters.method) val cipher = new Aead(symmetricParameters.key.toArrayUnsafe) if (aeadOptions.useAes) cipher.useAesGcm() @@ -59,7 +59,7 @@ private[libsodium] final class AEADCipherModule(defaultOptions: AEADCipherOption def decrypt(data: ByteString, parameters: EncryptionParameters): ByteString = { val symmetricParameters = EncryptionParameters.symmetric(parameters) - val aeadOptions = AEADCipherOptions(symmetricParameters.method) + val aeadOptions = AEADCipherOptions(symmetricParameters.method) val cipher = new Aead(symmetricParameters.key.toArrayUnsafe) if (aeadOptions.useAes) cipher.useAesGcm() diff --git a/crypto/libsodium/src/main/scala/com/karasiq/shadowcloud/crypto/libsodium/symmetric/ChaCha20Module.scala b/crypto/libsodium/src/main/scala/com/karasiq/shadowcloud/crypto/libsodium/symmetric/ChaCha20Module.scala index 3e981d00..d64efd51 100644 --- a/crypto/libsodium/src/main/scala/com/karasiq/shadowcloud/crypto/libsodium/symmetric/ChaCha20Module.scala +++ b/crypto/libsodium/src/main/scala/com/karasiq/shadowcloud/crypto/libsodium/symmetric/ChaCha20Module.scala @@ -1,12 +1,10 @@ package com.karasiq.shadowcloud.crypto.libsodium.symmetric - - import com.karasiq.shadowcloud.model.crypto.EncryptionMethod import org.abstractj.kalium.NaCl.Sodium private[libsodium] object ChaCha20Module extends SymmetricConstants { - val KeyBytes = Sodium.CRYPTO_STREAM_CHACHA20_KEYBYTES + val KeyBytes = Sodium.CRYPTO_STREAM_CHACHA20_KEYBYTES val NonceBytes = Sodium.CRYPTO_STREAM_CHACHA20_NONCEBYTES def apply(method: EncryptionMethod = EncryptionMethod("ChaCha20", KeyBytes * 8)): ChaCha20Module = { @@ -15,10 +13,9 @@ private[libsodium] object ChaCha20Module extends SymmetricConstants { } private[libsodium] final class ChaCha20Module(method: EncryptionMethod) - extends StreamCipherModule(method, ChaCha20Module.KeyBytes, ChaCha20Module.NonceBytes) { + extends StreamCipherModule(method, ChaCha20Module.KeyBytes, ChaCha20Module.NonceBytes) { protected def process(key: Array[Byte], nonce: Array[Byte], inArray: Array[Byte], outArray: Array[Byte]): Unit = { sodium.crypto_stream_chacha20_xor(outArray, inArray, inArray.length, nonce, key) } } - diff --git a/crypto/libsodium/src/main/scala/com/karasiq/shadowcloud/crypto/libsodium/symmetric/Salsa20Module.scala b/crypto/libsodium/src/main/scala/com/karasiq/shadowcloud/crypto/libsodium/symmetric/Salsa20Module.scala index 4794e385..feacdf60 100644 --- a/crypto/libsodium/src/main/scala/com/karasiq/shadowcloud/crypto/libsodium/symmetric/Salsa20Module.scala +++ b/crypto/libsodium/src/main/scala/com/karasiq/shadowcloud/crypto/libsodium/symmetric/Salsa20Module.scala @@ -5,7 +5,7 @@ import org.abstractj.kalium.NaCl.Sodium import com.karasiq.shadowcloud.model.crypto.EncryptionMethod private[libsodium] object Salsa20Module extends SymmetricConstants { - val KeyBytes = Sodium.CRYPTO_STREAM_SALSA20_KEYBYTES + val KeyBytes = Sodium.CRYPTO_STREAM_SALSA20_KEYBYTES val NonceBytes = Sodium.CRYPTO_STREAM_SALSA20_NONCEBYTES def apply(method: EncryptionMethod = EncryptionMethod("Salsa20", KeyBytes * 8)): Salsa20Module = { @@ -14,7 +14,7 @@ private[libsodium] object Salsa20Module extends SymmetricConstants { } private[libsodium] final class Salsa20Module(method: EncryptionMethod) - extends StreamCipherModule(method, Salsa20Module.KeyBytes, Salsa20Module.NonceBytes) { + extends StreamCipherModule(method, Salsa20Module.KeyBytes, Salsa20Module.NonceBytes) { protected def process(key: Array[Byte], nonce: Array[Byte], inArray: Array[Byte], outArray: Array[Byte]): Unit = { sodium.crypto_stream_salsa20_xor(outArray, inArray, inArray.length, nonce, key) diff --git a/crypto/libsodium/src/main/scala/com/karasiq/shadowcloud/crypto/libsodium/symmetric/SecretBoxModule.scala b/crypto/libsodium/src/main/scala/com/karasiq/shadowcloud/crypto/libsodium/symmetric/SecretBoxModule.scala index cdc18891..f9af6516 100644 --- a/crypto/libsodium/src/main/scala/com/karasiq/shadowcloud/crypto/libsodium/symmetric/SecretBoxModule.scala +++ b/crypto/libsodium/src/main/scala/com/karasiq/shadowcloud/crypto/libsodium/symmetric/SecretBoxModule.scala @@ -8,7 +8,7 @@ import com.karasiq.shadowcloud.model.crypto.EncryptionMethod private[libsodium] object SecretBoxModule extends SymmetricConstants { val Algorithm = "XSalsa20/Poly1305" - val KeyBytes: Int = Sodium.CRYPTO_SECRETBOX_KEYBYTES + val KeyBytes: Int = Sodium.CRYPTO_SECRETBOX_KEYBYTES val NonceBytes: Int = Sodium.CRYPTO_SECRETBOX_NONCEBYTES def apply(method: EncryptionMethod = EncryptionMethod(Algorithm, 256)): SecretBoxModule = { @@ -16,10 +16,9 @@ private[libsodium] object SecretBoxModule extends SymmetricConstants { } } -private[libsodium] final class SecretBoxModule(val method: EncryptionMethod) - extends SymmetricCipherModule with SymmetricCipherAtomic { +private[libsodium] final class SecretBoxModule(val method: EncryptionMethod) extends SymmetricCipherModule with SymmetricCipherAtomic { - protected val keySize: Int = SecretBoxModule.KeyBytes + protected val keySize: Int = SecretBoxModule.KeyBytes protected val nonceSize: Int = SecretBoxModule.NonceBytes protected def encrypt(data: Array[Byte], key: Array[Byte], nonce: Array[Byte]): Array[Byte] = { diff --git a/crypto/libsodium/src/main/scala/com/karasiq/shadowcloud/crypto/libsodium/symmetric/StreamCipherModule.scala b/crypto/libsodium/src/main/scala/com/karasiq/shadowcloud/crypto/libsodium/symmetric/StreamCipherModule.scala index 096162c6..ddf9f421 100644 --- a/crypto/libsodium/src/main/scala/com/karasiq/shadowcloud/crypto/libsodium/symmetric/StreamCipherModule.scala +++ b/crypto/libsodium/src/main/scala/com/karasiq/shadowcloud/crypto/libsodium/symmetric/StreamCipherModule.scala @@ -3,10 +3,9 @@ package com.karasiq.shadowcloud.crypto.libsodium.symmetric import com.karasiq.shadowcloud.crypto._ import com.karasiq.shadowcloud.model.crypto.EncryptionMethod -private[libsodium] abstract class StreamCipherModule(val method: EncryptionMethod, - protected val keySize: Int, - protected val nonceSize: Int) - extends SymmetricCipherModule with OnlyStreamEncryptionModule { +private[libsodium] abstract class StreamCipherModule(val method: EncryptionMethod, protected val keySize: Int, protected val nonceSize: Int) + extends SymmetricCipherModule + with OnlyStreamEncryptionModule { protected def process(key: Array[Byte], nonce: Array[Byte], inArray: Array[Byte], outArray: Array[Byte]): Unit @@ -22,8 +21,8 @@ private[libsodium] abstract class StreamCipherModule(val method: EncryptionMetho } private[libsodium] trait StreamCipherStreamer extends EncryptionModuleStreamer with SymmetricCipherStreaming { - protected var encrypt = true - protected var key: Array[Byte] = Array.emptyByteArray + protected var encrypt = true + protected var key: Array[Byte] = Array.emptyByteArray protected var nonce: Array[Byte] = Array.emptyByteArray protected def process(inArray: Array[Byte], outArray: Array[Byte]): Unit @@ -39,4 +38,4 @@ private[libsodium] trait StreamCipherStreamer extends EncryptionModuleStreamer w process(data, outArray) outArray } -} \ No newline at end of file +} diff --git a/crypto/libsodium/src/main/scala/com/karasiq/shadowcloud/crypto/libsodium/symmetric/SymmetricCipherModule.scala b/crypto/libsodium/src/main/scala/com/karasiq/shadowcloud/crypto/libsodium/symmetric/SymmetricCipherModule.scala index 4528ec89..36ff7a94 100644 --- a/crypto/libsodium/src/main/scala/com/karasiq/shadowcloud/crypto/libsodium/symmetric/SymmetricCipherModule.scala +++ b/crypto/libsodium/src/main/scala/com/karasiq/shadowcloud/crypto/libsodium/symmetric/SymmetricCipherModule.scala @@ -1,7 +1,5 @@ package com.karasiq.shadowcloud.crypto.libsodium.symmetric - - import akka.util.ByteString import com.karasiq.shadowcloud.crypto._ import com.karasiq.shadowcloud.crypto.libsodium.internal.LSUtils @@ -16,14 +14,14 @@ private[libsodium] object SymmetricCipherModule { } private[libsodium] trait SymmetricCipherModule extends EncryptionModule { - protected final val sodium = NaCl.sodium() + protected final val sodium = NaCl.sodium() protected final val secureRandom = LSUtils.createSecureRandom() protected val keySize: Int protected val nonceSize: Int def createParameters(): EncryptionParameters = { - val key = secureRandom.randomBytes(keySize) + val key = secureRandom.randomBytes(keySize) val nonce = secureRandom.randomBytes(nonceSize) SymmetricEncryptionParameters(method, ByteString.fromArrayUnsafe(key), ByteString.fromArrayUnsafe(nonce)) } diff --git a/crypto/libsodium/src/main/scala/com/karasiq/shadowcloud/crypto/libsodium/symmetric/XSalsa20Module.scala b/crypto/libsodium/src/main/scala/com/karasiq/shadowcloud/crypto/libsodium/symmetric/XSalsa20Module.scala index 6742f199..b0975681 100644 --- a/crypto/libsodium/src/main/scala/com/karasiq/shadowcloud/crypto/libsodium/symmetric/XSalsa20Module.scala +++ b/crypto/libsodium/src/main/scala/com/karasiq/shadowcloud/crypto/libsodium/symmetric/XSalsa20Module.scala @@ -5,7 +5,7 @@ import org.abstractj.kalium.NaCl.Sodium import com.karasiq.shadowcloud.model.crypto.EncryptionMethod private[libsodium] object XSalsa20Module extends SymmetricConstants { - val KeyBytes = Sodium.CRYPTO_STREAM_KEYBYTES + val KeyBytes = Sodium.CRYPTO_STREAM_KEYBYTES val NonceBytes = Sodium.CRYPTO_STREAM_NONCEBYTES def apply(method: EncryptionMethod = EncryptionMethod("XSalsa20", KeyBytes * 8)): XSalsa20Module = { @@ -14,7 +14,7 @@ private[libsodium] object XSalsa20Module extends SymmetricConstants { } private[libsodium] final class XSalsa20Module(method: EncryptionMethod) - extends StreamCipherModule(method, XSalsa20Module.KeyBytes, XSalsa20Module.NonceBytes) { + extends StreamCipherModule(method, XSalsa20Module.KeyBytes, XSalsa20Module.NonceBytes) { protected def process(key: Array[Byte], nonce: Array[Byte], inArray: Array[Byte], outArray: Array[Byte]): Unit = { sodium.crypto_stream_xor(outArray, inArray, inArray.length, nonce, key) diff --git a/crypto/libsodium/src/test/scala/com/karasiq/shadowcloud/crypto/libsodium/test/LibSodiumTest.scala b/crypto/libsodium/src/test/scala/com/karasiq/shadowcloud/crypto/libsodium/test/LibSodiumTest.scala index c2ccf45d..5b18ce94 100644 --- a/crypto/libsodium/src/test/scala/com/karasiq/shadowcloud/crypto/libsodium/test/LibSodiumTest.scala +++ b/crypto/libsodium/src/test/scala/com/karasiq/shadowcloud/crypto/libsodium/test/LibSodiumTest.scala @@ -1,7 +1,5 @@ package com.karasiq.shadowcloud.crypto.libsodium.test - - import akka.util.ByteString import com.karasiq.common.encoding.HexString import com.karasiq.shadowcloud.config.ConfigProps @@ -17,7 +15,9 @@ import org.scalatest.{FlatSpec, Matchers} class LibSodiumTest extends FlatSpec with Matchers { val testVectors = CryptoTestVectors("libsodium") - val testData = ByteString("# First, make a nonce: A single-use value never repeated under the same key\n# The nonce isn't secret, and can be sent with the ciphertext.\n# The cipher instance has a nonce_bytes method for determining how many bytes should be in a nonce") + val testData = ByteString( + "# First, make a nonce: A single-use value never repeated under the same key\n# The nonce isn't secret, and can be sent with the ciphertext.\n# The cipher instance has a nonce_bytes method for determining how many bytes should be in a nonce" + ) if (LSUtils.isLibraryAvailable) { // Encryption @@ -35,19 +35,29 @@ class LibSodiumTest extends FlatSpec with Matchers { } // Hashes - testHashing("SHA256", MultiPartHashModule.SHA256(), - "e3fc39605cd8e9245ed8cb41e2730c940e6026b9d2f72ead3b0f2d271e2290e0") - - testHashing("SHA512", MultiPartHashModule.SHA512(), - "11bba64289c2fefc6caf753cc14fd3b914663f0035b0e2135bb29fc5159f9e99ddc57c577146688f4b64cfae09d9be933c22b17eb4a08cdb92e2c1d68efa0f59") - - testHashing("Blake2b", Blake2bModule(), - "824396f4585a22b2c4b36df76f55e669d4edfb423970071b6b616ce454a95400") - - testHashing("Blake2b+key", Blake2bModule(HashingMethod("Blake2b", config = ConfigProps("digest-key" → "824396f4585a22b2c4b36df76f55e669d4edfb423970071b6b616ce454a95400"))), "717fc02f9817eb24cc3f9e803fddebf81fc18b415537b1ea9bb08691215d162d") - - testHashing("Blake2b-512", Blake2bModule(HashingMethod("Blake2b", config = ConfigProps("digest-size" → 512))), - "9f84251be0c325ad771696302e9ed3cd174f84ffdd0b8de49664e9a3ea934b89a4d008581cd5803b80b3284116174b3c4a79a5029996eb59edc1fbacfd18204e") + testHashing("SHA256", MultiPartHashModule.SHA256(), "e3fc39605cd8e9245ed8cb41e2730c940e6026b9d2f72ead3b0f2d271e2290e0") + + testHashing( + "SHA512", + MultiPartHashModule.SHA512(), + "11bba64289c2fefc6caf753cc14fd3b914663f0035b0e2135bb29fc5159f9e99ddc57c577146688f4b64cfae09d9be933c22b17eb4a08cdb92e2c1d68efa0f59" + ) + + testHashing("Blake2b", Blake2bModule(), "824396f4585a22b2c4b36df76f55e669d4edfb423970071b6b616ce454a95400") + + testHashing( + "Blake2b+key", + Blake2bModule( + HashingMethod("Blake2b", config = ConfigProps("digest-key" → "824396f4585a22b2c4b36df76f55e669d4edfb423970071b6b616ce454a95400")) + ), + "717fc02f9817eb24cc3f9e803fddebf81fc18b415537b1ea9bb08691215d162d" + ) + + testHashing( + "Blake2b-512", + Blake2bModule(HashingMethod("Blake2b", config = ConfigProps("digest-size" → 512))), + "9f84251be0c325ad771696302e9ed3cd174f84ffdd0b8de49664e9a3ea934b89a4d008581cd5803b80b3284116174b3c4a79a5029996eb59edc1fbacfd18204e" + ) // Signatures testSignature("Ed25519", CryptoSignModule()) @@ -66,7 +76,7 @@ class LibSodiumTest extends FlatSpec with Matchers { it should "encrypt data" in { val parameters = module.createParameters() - val encrypted = module.encrypt(testData, parameters) + val encrypted = module.encrypt(testData, parameters) encrypted.length should be >= testData.length val decrypted = module.decrypt(encrypted, parameters) decrypted shouldBe testData @@ -75,7 +85,7 @@ class LibSodiumTest extends FlatSpec with Matchers { it should "decrypt test vector" in { val (parameters, plain, encrypted) = testVectors.load(name) - val decrypted = module.decrypt(encrypted, parameters) + val decrypted = module.decrypt(encrypted, parameters) decrypted shouldBe plain val encrypted1 = module.encrypt(plain, parameters) encrypted1 shouldBe encrypted @@ -84,14 +94,14 @@ class LibSodiumTest extends FlatSpec with Matchers { private[this] def testAsymmetricEncryption(name: String, module: EncryptionModule): Unit = { s"$name module" should "generate key" in { - val parameters = EncryptionParameters.asymmetric(module.createParameters()) + val parameters = EncryptionParameters.asymmetric(module.createParameters()) val parameters1 = EncryptionParameters.asymmetric(module.updateParameters(parameters)) parameters1 shouldBe parameters } it should "encrypt data" in { val parameters = module.createParameters() - val encrypted = module.encrypt(testData, parameters) + val encrypted = module.encrypt(testData, parameters) encrypted.length should be >= testData.length val decrypted = module.decrypt(encrypted, parameters) decrypted shouldBe testData @@ -100,7 +110,7 @@ class LibSodiumTest extends FlatSpec with Matchers { it should "decrypt test vector" in { val (parameters, plain, encrypted) = testVectors.load(name) - val decrypted = module.decrypt(encrypted, parameters) + val decrypted = module.decrypt(encrypted, parameters) decrypted shouldBe plain } } @@ -124,9 +134,9 @@ class LibSodiumTest extends FlatSpec with Matchers { it should "sign data" in { val parameters = module.createParameters() - val signature = module.sign(testData, parameters) + val signature = module.sign(testData, parameters) module.verify(testData, signature, parameters) shouldBe true - module.verify(testData, signature.reverse, parameters) shouldBe false + module.verify(testData, signature.reverse, parameters) shouldBe false } } } diff --git a/crypto/parent/src/main/scala/com/karasiq/shadowcloud/crypto/CryptoModule.scala b/crypto/parent/src/main/scala/com/karasiq/shadowcloud/crypto/CryptoModule.scala index 4f5cc887..9ee92f47 100644 --- a/crypto/parent/src/main/scala/com/karasiq/shadowcloud/crypto/CryptoModule.scala +++ b/crypto/parent/src/main/scala/com/karasiq/shadowcloud/crypto/CryptoModule.scala @@ -1,7 +1,5 @@ package com.karasiq.shadowcloud.crypto - - import akka.util.ByteString import com.karasiq.shadowcloud.model.crypto.CryptoMethod diff --git a/crypto/parent/src/main/scala/com/karasiq/shadowcloud/crypto/EncryptionModule.scala b/crypto/parent/src/main/scala/com/karasiq/shadowcloud/crypto/EncryptionModule.scala index 86ad3002..24fb59c3 100644 --- a/crypto/parent/src/main/scala/com/karasiq/shadowcloud/crypto/EncryptionModule.scala +++ b/crypto/parent/src/main/scala/com/karasiq/shadowcloud/crypto/EncryptionModule.scala @@ -1,7 +1,5 @@ package com.karasiq.shadowcloud.crypto - - import akka.util.ByteString import com.karasiq.shadowcloud.model.crypto.{EncryptionMethod, EncryptionParameters} diff --git a/crypto/parent/src/main/scala/com/karasiq/shadowcloud/crypto/HashingModule.scala b/crypto/parent/src/main/scala/com/karasiq/shadowcloud/crypto/HashingModule.scala index 30499ac5..45bbbe40 100644 --- a/crypto/parent/src/main/scala/com/karasiq/shadowcloud/crypto/HashingModule.scala +++ b/crypto/parent/src/main/scala/com/karasiq/shadowcloud/crypto/HashingModule.scala @@ -1,7 +1,5 @@ package com.karasiq.shadowcloud.crypto - - import akka.util.ByteString import com.karasiq.shadowcloud.model.crypto.HashingMethod diff --git a/crypto/parent/src/main/scala/com/karasiq/shadowcloud/crypto/SignModule.scala b/crypto/parent/src/main/scala/com/karasiq/shadowcloud/crypto/SignModule.scala index 02aaf3df..c9504849 100644 --- a/crypto/parent/src/main/scala/com/karasiq/shadowcloud/crypto/SignModule.scala +++ b/crypto/parent/src/main/scala/com/karasiq/shadowcloud/crypto/SignModule.scala @@ -1,7 +1,5 @@ package com.karasiq.shadowcloud.crypto - - import akka.util.ByteString import com.karasiq.shadowcloud.model.crypto.{SignMethod, SignParameters} diff --git a/crypto/parent/src/main/scala/com/karasiq/shadowcloud/providers/CryptoProvider.scala b/crypto/parent/src/main/scala/com/karasiq/shadowcloud/providers/CryptoProvider.scala index a998dfab..223209b5 100644 --- a/crypto/parent/src/main/scala/com/karasiq/shadowcloud/providers/CryptoProvider.scala +++ b/crypto/parent/src/main/scala/com/karasiq/shadowcloud/providers/CryptoProvider.scala @@ -1,21 +1,19 @@ package com.karasiq.shadowcloud.providers - - import com.karasiq.shadowcloud.crypto._ import com.karasiq.shadowcloud.model.crypto.{EncryptionMethod, HashingMethod, SignMethod} abstract class CryptoProvider extends ModuleProvider { - type HashingPF = PartialFunction[HashingMethod, HashingModule] + type HashingPF = PartialFunction[HashingMethod, HashingModule] type EncryptionPF = PartialFunction[EncryptionMethod, EncryptionModule] - type SignPF = PartialFunction[SignMethod, SignModule] + type SignPF = PartialFunction[SignMethod, SignModule] def hashingAlgorithms: Set[String] = Set.empty - def hashing: HashingPF = PartialFunction.empty + def hashing: HashingPF = PartialFunction.empty def encryptionAlgorithms: Set[String] = Set.empty - def encryption: EncryptionPF = PartialFunction.empty + def encryption: EncryptionPF = PartialFunction.empty def signingAlgorithms: Set[String] = Set.empty - def signing: SignPF = PartialFunction.empty + def signing: SignPF = PartialFunction.empty } diff --git a/desktop-app/src/main/scala/com/karasiq/shadowcloud/desktop/SCDesktopConfig.scala b/desktop-app/src/main/scala/com/karasiq/shadowcloud/desktop/SCDesktopConfig.scala index 900d46de..96eccc97 100644 --- a/desktop-app/src/main/scala/com/karasiq/shadowcloud/desktop/SCDesktopConfig.scala +++ b/desktop-app/src/main/scala/com/karasiq/shadowcloud/desktop/SCDesktopConfig.scala @@ -39,7 +39,7 @@ object SCDesktopConfig { "shadowcloud.conf" ) ++ sys.props.get("shadowcloud.external-config")).map(new File(_)).filter(_.isFile) - val fileConfig = files.foldLeft(ConfigUtils.emptyConfig) { (conf, f) => + val fileConfig = files.foldLeft(ConfigUtils.emptyConfig) { (conf, f) ⇒ ConfigFactory.parseFile(f).withFallback(conf) } diff --git a/drive/core/src/main/scala/com/karasiq/shadowcloud/drive/FileIOScheduler.scala b/drive/core/src/main/scala/com/karasiq/shadowcloud/drive/FileIOScheduler.scala index 0d509cea..50658367 100644 --- a/drive/core/src/main/scala/com/karasiq/shadowcloud/drive/FileIOScheduler.scala +++ b/drive/core/src/main/scala/com/karasiq/shadowcloud/drive/FileIOScheduler.scala @@ -179,10 +179,9 @@ class FileIOScheduler(config: SCDriveConfig, regionId: RegionId, file: File) ext val appendsRange = ChunkRanges.Range(chunksEnd, range.end) Source .fromIterator(() ⇒ dataUtils.pendingAppends(currentWrites, chunksEnd)) - .map { - case (range, chunk) ⇒ - val selectedRange = range.relativeTo(appendsRange).fitToSize(appendsRange.size) - selectedRange.slice(chunk.data.plain) + .map { case (range, chunk) ⇒ + val selectedRange = range.relativeTo(appendsRange).fitToSize(appendsRange.size) + selectedRange.slice(chunk.data.plain) } .named("scDriveReadAppends") } @@ -412,7 +411,8 @@ class FileIOScheduler(config: SCDriveConfig, regionId: RegionId, file: File) ext case Flush ⇒ actorState.pendingFlush.addWaiter( NotUsed, - sender(), { () ⇒ + sender(), + { () ⇒ log.info("Flushing writes: {}", dataState.pendingWrites) // MemorySize(dataState.pendingWrites.map(_.range.size).sum) Flush.wrapFuture(file, dataIO.flush()).pipeTo(self) } diff --git a/drive/core/src/main/scala/com/karasiq/shadowcloud/drive/SCDriveExtension.scala b/drive/core/src/main/scala/com/karasiq/shadowcloud/drive/SCDriveExtension.scala index 201aa1b2..ccdbcf54 100644 --- a/drive/core/src/main/scala/com/karasiq/shadowcloud/drive/SCDriveExtension.scala +++ b/drive/core/src/main/scala/com/karasiq/shadowcloud/drive/SCDriveExtension.scala @@ -20,6 +20,6 @@ object SCDrive extends ExtensionId[SCDriveExtension] with ExtensionIdProvider { } class SCDriveExtension(system: ExtendedActorSystem) extends Extension { - val config = SCDriveConfig(system.settings.config.getConfig("shadowcloud.drive")) + val config = SCDriveConfig(system.settings.config.getConfig("shadowcloud.drive")) lazy val dispatcher = system.actorOf(VirtualFSDispatcher.props(config), "sc-drive-dispatcher") -} \ No newline at end of file +} diff --git a/drive/core/src/main/scala/com/karasiq/shadowcloud/drive/VirtualFSDispatcher.scala b/drive/core/src/main/scala/com/karasiq/shadowcloud/drive/VirtualFSDispatcher.scala index 0ae25fb3..43a6ff92 100644 --- a/drive/core/src/main/scala/com/karasiq/shadowcloud/drive/VirtualFSDispatcher.scala +++ b/drive/core/src/main/scala/com/karasiq/shadowcloud/drive/VirtualFSDispatcher.scala @@ -126,10 +126,11 @@ class VirtualFSDispatcher(config: SCDriveConfig) extends Actor with ActorLogging for { regions ← sc.ops.supervisor.getSnapshot() regionIsActive = regions.regions.get(regionId).exists(_.actorState.isActive) - folder ← if (regionIsActive) - sc.ops.region.getFolder(regionId, regionPath) - else - Future.successful(Folder.create(regionPath)) + folder ← + if (regionIsActive) + sc.ops.region.getFolder(regionId, regionPath) + else + Future.successful(Folder.create(regionPath)) files ← Future.sequence(virtualFiles) fileNameSet = files.flatten.map(_.path.name).toSet } yield folder.copy(files = folder.files.filterNot(f ⇒ fileNameSet.contains(f.path.name)) ++ files.flatten) diff --git a/drive/core/src/main/scala/com/karasiq/shadowcloud/drive/utils/ChunkPatch.scala b/drive/core/src/main/scala/com/karasiq/shadowcloud/drive/utils/ChunkPatch.scala index ffa0ea8a..b6dff619 100644 --- a/drive/core/src/main/scala/com/karasiq/shadowcloud/drive/utils/ChunkPatch.scala +++ b/drive/core/src/main/scala/com/karasiq/shadowcloud/drive/utils/ChunkPatch.scala @@ -31,11 +31,10 @@ private[drive] final case class ChunkPatchList(patches: Seq[ChunkPatch]) { } def patchChunk(dataRange: ChunkRanges.Range, data: ByteString) = { - patches.foldLeft(data) { - case (data, write) ⇒ - val relRange = write.range.relativeTo(dataRange) - val offset = dataRange.relativeTo(write.range) - ChunkRanges.Range.patch(data, relRange, offset.slice(write.data)) + patches.foldLeft(data) { case (data, write) ⇒ + val relRange = write.range.relativeTo(dataRange) + val offset = dataRange.relativeTo(write.range) + ChunkRanges.Range.patch(data, relRange, offset.slice(write.data)) } } } diff --git a/drive/core/src/main/scala/com/karasiq/shadowcloud/drive/utils/PendingChunkIO.scala b/drive/core/src/main/scala/com/karasiq/shadowcloud/drive/utils/PendingChunkIO.scala index e03699ef..e6f6441b 100644 --- a/drive/core/src/main/scala/com/karasiq/shadowcloud/drive/utils/PendingChunkIO.scala +++ b/drive/core/src/main/scala/com/karasiq/shadowcloud/drive/utils/PendingChunkIO.scala @@ -11,6 +11,5 @@ private[drive] sealed trait PendingChunkIO { private[drive] object PendingChunkIO { final case class Rewrite(range: ChunkRanges.Range, chunk: Chunk, patches: ChunkPatchList) extends PendingChunkIO - final case class Append(range: ChunkRanges.Range, newData: ByteString) extends PendingChunkIO + final case class Append(range: ChunkRanges.Range, newData: ByteString) extends PendingChunkIO } - diff --git a/drive/core/src/main/scala/com/karasiq/shadowcloud/drive/utils/WritesOptimizeStage.scala b/drive/core/src/main/scala/com/karasiq/shadowcloud/drive/utils/WritesOptimizeStage.scala index eb3c8cd3..62d21327 100644 --- a/drive/core/src/main/scala/com/karasiq/shadowcloud/drive/utils/WritesOptimizeStage.scala +++ b/drive/core/src/main/scala/com/karasiq/shadowcloud/drive/utils/WritesOptimizeStage.scala @@ -15,14 +15,14 @@ private[drive] object WritesOptimizeStage { } private[drive] class WritesOptimizeStage(chunkSize: Long) extends GraphStage[FlowShape[PendingChunkIO, PendingChunkIO]] { - val inlet = Inlet[PendingChunkIO]("WritesFoldStage.in") + val inlet = Inlet[PendingChunkIO]("WritesFoldStage.in") val outlet = Outlet[PendingChunkIO]("WritesFoldStage.out") - val shape = FlowShape(inlet, outlet) + val shape = FlowShape(inlet, outlet) def createLogic(inheritedAttributes: Attributes) = new GraphStageLogic(shape) with InHandler with OutHandler { - var appendOffset = 0L + var appendOffset = 0L var currentOffset = 0L - var appendBuffer = ByteString.empty + var appendBuffer = ByteString.empty def addAppendBytes(bs: ByteString): Unit = { if (appendBuffer.isEmpty) appendOffset = currentOffset @@ -34,7 +34,7 @@ private[drive] class WritesOptimizeStage(chunkSize: Long) extends GraphStage[Flo if (appendBuffer.isEmpty || appendBuffer.length < threshold) return val (drop, keep) = appendBuffer.splitAt(splitSize) - val appendOp = PendingChunkIO.Append(ChunkRanges.Range(appendOffset, appendOffset + drop.length), drop) + val appendOp = PendingChunkIO.Append(ChunkRanges.Range(appendOffset, appendOffset + drop.length), drop) appendBuffer = keep appendOffset += drop.length @@ -53,7 +53,8 @@ private[drive] class WritesOptimizeStage(chunkSize: Long) extends GraphStage[Flo def onPush(): Unit = { val element = grab(inlet) element match { - case PendingChunkIO.Rewrite(range, _, patches) if element.range.start == currentOffset && range.size < chunkSize && patches.canReplace(range.size) ⇒ + case PendingChunkIO.Rewrite(range, _, patches) + if element.range.start == currentOffset && range.size < chunkSize && patches.canReplace(range.size) ⇒ addAppendBytes(patches.toBytes(range.size.toInt)) emitFullAppends() @@ -74,8 +75,7 @@ private[drive] class WritesOptimizeStage(chunkSize: Long) extends GraphStage[Flo emitAppends() super.onUpstreamFinish() } - + setHandlers(inlet, outlet, this) } } - diff --git a/drive/core/src/test/scala/com/karasiq/shadowcloud/drive/test/FileIOSchedulerTest.scala b/drive/core/src/test/scala/com/karasiq/shadowcloud/drive/test/FileIOSchedulerTest.scala index d4255ebe..9d32cc65 100644 --- a/drive/core/src/test/scala/com/karasiq/shadowcloud/drive/test/FileIOSchedulerTest.scala +++ b/drive/core/src/test/scala/com/karasiq/shadowcloud/drive/test/FileIOSchedulerTest.scala @@ -74,16 +74,15 @@ final class FileIOSchedulerTest extends SCExtensionSpec with FlatSpecLike { flushResult.writes shouldBe Seq(write: ChunkPatch) flushResult.ops.sortBy(_.range.start) match { case ChunkIOOperation.ChunkRewritten(ChunkRanges.Range(0, 20), oldChunk, chunk) +: - ChunkIOOperation.ChunkAppended(ChunkRanges.Range(20, 30), newChunk) +: Nil ⇒ + ChunkIOOperation.ChunkAppended(ChunkRanges.Range(20, 30), newChunk) +: Nil ⇒ oldChunk.checksum.size shouldBe 20 chunk.checksum.size shouldBe 20 newChunk.checksum.size shouldBe 10 } - testChunks { - case chunk +: newChunk +: Nil ⇒ - chunk.checksum.size shouldBe 20 - newChunk.checksum.size shouldBe 10 + testChunks { case chunk +: newChunk +: Nil ⇒ + chunk.checksum.size shouldBe 20 + newChunk.checksum.size shouldBe 10 } testRead(testData ++ zeroes ++ testData) @@ -102,9 +101,8 @@ final class FileIOSchedulerTest extends SCExtensionSpec with FlatSpecLike { newChunk.checksum.size shouldBe 40 } - testChunks { - case newChunk +: Nil ⇒ - newChunk.checksum.size shouldBe 40 + testChunks { case newChunk +: Nil ⇒ + newChunk.checksum.size shouldBe 40 } testRead(testData ++ zeroes ++ testData ++ testData) @@ -122,9 +120,8 @@ final class FileIOSchedulerTest extends SCExtensionSpec with FlatSpecLike { newChunk.checksum.size shouldBe 25 } - testChunks { - case newChunk +: Nil ⇒ - newChunk.checksum.size shouldBe 25 + testChunks { case newChunk +: Nil ⇒ + newChunk.checksum.size shouldBe 25 } testRead(testData ++ zeroes ++ testData.take(5)) } diff --git a/drive/fuse/src/main/scala/com/karasiq/shadowcloud/drive/fuse/SCFileSystem.scala b/drive/fuse/src/main/scala/com/karasiq/shadowcloud/drive/fuse/SCFileSystem.scala index ba7d810a..34fe4dcc 100644 --- a/drive/fuse/src/main/scala/com/karasiq/shadowcloud/drive/fuse/SCFileSystem.scala +++ b/drive/fuse/src/main/scala/com/karasiq/shadowcloud/drive/fuse/SCFileSystem.scala @@ -103,26 +103,27 @@ class SCFileSystem(config: SCDriveConfig, fsDispatcher: ActorRef, actorSystem: A mount(Paths.get(mountPath), blocking, debug, Array("-o", options.mkString(","))) } - protected def dispatch[T](message: AnyRef, status: MessageStatus[_, T], critical: Boolean = false, handle: Long = 0)( - implicit timeout: Timeout + protected def dispatch[T](message: AnyRef, status: MessageStatus[_, T], critical: Boolean = false, handle: Long = 0)(implicit + timeout: Timeout ): T = { // if (critical) log.info(s"IO operation requested: $message") def getResult() = Await.result(status.unwrapFuture(fsDispatcher ? message), timeout.duration) val start = System.nanoTime() - val result = try { - if (critical && settings.synchronizedMode) { - val fh = if (handle == 0) null else fileHandles.get(handle).orNull - if (fh eq null) synchronized(getResult()) - else fh.synchronized(getResult()) - } else { - getResult() + val result = + try { + if (critical && settings.synchronizedMode) { + val fh = if (handle == 0) null else fileHandles.get(handle).orNull + if (fh eq null) synchronized(getResult()) + else fh.synchronized(getResult()) + } else { + getResult() + } + } catch { + case NonFatal(error) ⇒ + if (critical || settings.debug) log.error(error, "IO operation failed: {}", message) + throw error } - } catch { - case NonFatal(error) ⇒ - if (critical || settings.debug) log.error(error, "IO operation failed: {}", message) - throw error - } val elapsed = (System.nanoTime() - start).nanos if (settings.debug || elapsed >= (5 seconds)) log.info("IO operation completed in {} ms: {} -> {}", elapsed.toMillis, message, result) result diff --git a/javafx/src/main/scala/com/karasiq/shadowcloud/javafx/JFXUtils.scala b/javafx/src/main/scala/com/karasiq/shadowcloud/javafx/JFXUtils.scala index c8f988ea..8d2235d3 100644 --- a/javafx/src/main/scala/com/karasiq/shadowcloud/javafx/JFXUtils.scala +++ b/javafx/src/main/scala/com/karasiq/shadowcloud/javafx/JFXUtils.scala @@ -10,11 +10,11 @@ private[javafx] object JFXUtils { getClass.getClassLoader.getResource(fileName).toString } - def runNow[T](f: => T): T = { + def runNow[T](f: ⇒ T): T = { val promise = Promise[T] Platform.runLater { try promise.success(f) - catch { case err: Throwable => promise.failure(err) } + catch { case err: Throwable ⇒ promise.failure(err) } } Await.result(promise.future, Duration.Inf) } diff --git a/javafx/src/main/scala/com/karasiq/shadowcloud/javafx/JavaFXContext.scala b/javafx/src/main/scala/com/karasiq/shadowcloud/javafx/JavaFXContext.scala index c4bb860e..8b9f2085 100644 --- a/javafx/src/main/scala/com/karasiq/shadowcloud/javafx/JavaFXContext.scala +++ b/javafx/src/main/scala/com/karasiq/shadowcloud/javafx/JavaFXContext.scala @@ -27,7 +27,7 @@ final class JavaFXContextExtension(system: ExtendedActorSystem) extends Extensio val initFuture = initPromise.future object app extends JFXApp { - stage = new PrimaryStage { stage => + stage = new PrimaryStage { stage ⇒ initStyle(StageStyle.Transparent) icons.add(new Image(JFXUtils.getResourcePath("sc-javafx/sc-icon.png"))) scene = new Scene { @@ -40,7 +40,8 @@ final class JavaFXContextExtension(system: ExtendedActorSystem) extends Extensio } addEventHandler( - WindowEvent.WINDOW_SHOWN, { _: WindowEvent => + WindowEvent.WINDOW_SHOWN, + { _: WindowEvent ⇒ import javafx.stage.Screen val screenBounds = Screen.getPrimary.getVisualBounds stage.x = (screenBounds.getWidth - stage.width()) / 2 diff --git a/javafx/src/main/scala/com/karasiq/shadowcloud/javafx/JavaFXUIProvider.scala b/javafx/src/main/scala/com/karasiq/shadowcloud/javafx/JavaFXUIProvider.scala index baf22172..aebedd35 100644 --- a/javafx/src/main/scala/com/karasiq/shadowcloud/javafx/JavaFXUIProvider.scala +++ b/javafx/src/main/scala/com/karasiq/shadowcloud/javafx/JavaFXUIProvider.scala @@ -6,7 +6,7 @@ import akka.actor.ActorSystem import com.karasiq.shadowcloud.ui.UIProvider private[javafx] final class JavaFXUIProvider(actorSystem: ActorSystem) extends UIProvider { - private[this] lazy val context = JavaFXContext(actorSystem) + private[this] lazy val context = JavaFXContext(actorSystem) @volatile private[this] var notificationPresent = false override def showErrorMessage(error: Throwable): Unit = { diff --git a/javafx/src/main/scala/com/karasiq/shadowcloud/javafx/PasswordDialog.scala b/javafx/src/main/scala/com/karasiq/shadowcloud/javafx/PasswordDialog.scala index c67153fd..c55ab0f6 100644 --- a/javafx/src/main/scala/com/karasiq/shadowcloud/javafx/PasswordDialog.scala +++ b/javafx/src/main/scala/com/karasiq/shadowcloud/javafx/PasswordDialog.scala @@ -49,13 +49,19 @@ private[javafx] class PasswordDialog(passwordId: String) extends Dialog[String] dialogPane().content = gridPane - dialogPane().scene().window().addEventHandler(WindowEvent.WINDOW_SHOWN, { e: WindowEvent => - import javafx.stage.{Screen, Window} - val window = e.getSource.asInstanceOf[Window] - val screenBounds = Screen.getPrimary.getVisualBounds - window.setX((screenBounds.getWidth - window.getWidth) / 2) - window.setY((screenBounds.getHeight - window.getHeight) / 2) - }) + dialogPane() + .scene() + .window() + .addEventHandler( + WindowEvent.WINDOW_SHOWN, + { e: WindowEvent ⇒ + import javafx.stage.{Screen, Window} + val window = e.getSource.asInstanceOf[Window] + val screenBounds = Screen.getPrimary.getVisualBounds + window.setX((screenBounds.getWidth - window.getWidth) / 2) + window.setY((screenBounds.getHeight - window.getHeight) / 2) + } + ) dialogPane().scene().window().asInstanceOf[Stage].setAlwaysOnTop(true) } diff --git a/larray-bytestring/src/main/scala/akka/util/BSLArray.scala b/larray-bytestring/src/main/scala/akka/util/BSLArray.scala index 45ccb65b..33db87a1 100644 --- a/larray-bytestring/src/main/scala/akka/util/BSLArray.scala +++ b/larray-bytestring/src/main/scala/akka/util/BSLArray.scala @@ -5,13 +5,13 @@ import xerial.larray.{LArray, LByteArray} object BSLArray { def toLArray(arr: Array[Byte]): LByteArray = { val lb = new LByteArray(arr.length) - for (i <- arr.indices) lb(i) = arr(i) + for (i ← arr.indices) lb(i) = arr(i) lb } def toHeapArray(lb: LArray[Byte]): Array[Byte] = { val heapArr = new Array[Byte](lb.length.ensuring(_.isValidInt).toInt) - for (i <- heapArr.indices) heapArr(i) = lb(i) + for (i ← heapArr.indices) heapArr(i) = lb(i) heapArr } } diff --git a/larray-bytestring/src/main/scala/akka/util/ByteIterator.scala b/larray-bytestring/src/main/scala/akka/util/ByteIterator.scala index a71b9dd2..827ceee2 100644 --- a/larray-bytestring/src/main/scala/akka/util/ByteIterator.scala +++ b/larray-bytestring/src/main/scala/akka/util/ByteIterator.scala @@ -1,6 +1,5 @@ -/** - * Copyright (C) 2009-2018 Lightbend Inc. - */ +/** Copyright (C) 2009-2018 Lightbend Inc. + */ package akka.util @@ -53,19 +52,20 @@ object ByteIterator { case that: ByteIterator ⇒ if (that.isEmpty) this else if (this.isEmpty) that - else that match { - case that: ByteArrayIterator ⇒ - if ((this.array eq that.array) && (this.until == that.from)) { - this.until = that.until - that.clear() - this - } else { - val result = MultiByteArrayIterator(List(this, that)) - this.clear() - result - } - case that: MultiByteArrayIterator ⇒ this ++: that - } + else + that match { + case that: ByteArrayIterator ⇒ + if ((this.array eq that.array) && (this.until == that.from)) { + this.until = that.until + that.clear() + this + } else { + val result = MultiByteArrayIterator(List(this, that)) + this.clear() + result + } + case that: MultiByteArrayIterator ⇒ this ++: that + } case _ ⇒ super.++(that) } @@ -91,14 +91,15 @@ object ByteIterator { final override def dropWhile(p: Byte ⇒ Boolean): this.type = { var stop = false while (!stop && hasNext) { - if (p(array(from))) { from = from + 1 } else { stop = true } + if (p(array(from))) { from = from + 1 } + else { stop = true } } this } final override def copyToArray[B >: Byte](xs: Array[B], start: Int, len: Int): Unit = { val n = 0 max ((xs.length - start) min this.len min len) - for (i <- 0 until n) xs(start + i) = array(from + i) + for (i ← 0 until n) xs(start + i) = array(from + i) this.drop(n) } @@ -125,25 +126,30 @@ object ByteIterator { array.toDirectByteBuffer.head.asReadOnlyBuffer() } - def getShorts(xs: Array[Short], offset: Int, n: Int)(implicit byteOrder: ByteOrder): this.type = - { wrappedByteBuffer(offset, n * 2).order(byteOrder).asShortBuffer.get(xs, offset, n); drop(2 * n) } + def getShorts(xs: Array[Short], offset: Int, n: Int)(implicit byteOrder: ByteOrder): this.type = { + wrappedByteBuffer(offset, n * 2).order(byteOrder).asShortBuffer.get(xs, offset, n); drop(2 * n) + } - def getInts(xs: Array[Int], offset: Int, n: Int)(implicit byteOrder: ByteOrder): this.type = - { wrappedByteBuffer(offset, n * 4).order(byteOrder).asIntBuffer.get(xs, offset, n); drop(4 * n) } + def getInts(xs: Array[Int], offset: Int, n: Int)(implicit byteOrder: ByteOrder): this.type = { + wrappedByteBuffer(offset, n * 4).order(byteOrder).asIntBuffer.get(xs, offset, n); drop(4 * n) + } - def getLongs(xs: Array[Long], offset: Int, n: Int)(implicit byteOrder: ByteOrder): this.type = - { wrappedByteBuffer(offset, n * 8).order(byteOrder).asLongBuffer.get(xs, offset, n); drop(8 * n) } + def getLongs(xs: Array[Long], offset: Int, n: Int)(implicit byteOrder: ByteOrder): this.type = { + wrappedByteBuffer(offset, n * 8).order(byteOrder).asLongBuffer.get(xs, offset, n); drop(8 * n) + } - def getFloats(xs: Array[Float], offset: Int, n: Int)(implicit byteOrder: ByteOrder): this.type = - { wrappedByteBuffer(offset, n * 4).order(byteOrder).asFloatBuffer.get(xs, offset, n); drop(4 * n) } + def getFloats(xs: Array[Float], offset: Int, n: Int)(implicit byteOrder: ByteOrder): this.type = { + wrappedByteBuffer(offset, n * 4).order(byteOrder).asFloatBuffer.get(xs, offset, n); drop(4 * n) + } - def getDoubles(xs: Array[Double], offset: Int, n: Int)(implicit byteOrder: ByteOrder): this.type = - { wrappedByteBuffer(offset, n * 8).order(byteOrder).asDoubleBuffer.get(xs, offset, n); drop(8 * n) } + def getDoubles(xs: Array[Double], offset: Int, n: Int)(implicit byteOrder: ByteOrder): this.type = { + wrappedByteBuffer(offset, n * 8).order(byteOrder).asDoubleBuffer.get(xs, offset, n); drop(8 * n) + } def copyToBuffer(buffer: ByteBuffer): Int = { val copyLength = math.min(buffer.remaining, len) if (copyLength > 0) { - for (i <- from until (from + copyLength)) buffer.put(array(i)) + for (i ← from until (from + copyLength)) buffer.put(array(i)) drop(copyLength) } copyLength @@ -249,7 +255,7 @@ object ByteIterator { /** For performance sensitive code, call take() directly on ByteString (it's optimised there) */ final override def take(n: Int): this.type = { - var rest = n + var rest = n val builder = new ListBuffer[ByteArrayIterator] while ((rest > 0) && !iterators.isEmpty) { current.take(rest) @@ -276,7 +282,7 @@ object ByteIterator { } else this final override def takeWhile(p: Byte ⇒ Boolean): this.type = { - var stop = false + var stop = false var builder = new ListBuffer[ByteArrayIterator] while (!stop && !iterators.isEmpty) { val lastLen = current.len @@ -298,7 +304,7 @@ object ByteIterator { } else this final override def copyToArray[B >: Byte](xs: Array[B], start: Int, len: Int): Unit = { - var pos = start + var pos = start var rest = len while ((rest > 0) && !iterators.isEmpty) { val n = 0 max ((xs.length - pos) min current.len min rest) @@ -326,8 +332,11 @@ object ByteIterator { } } - @tailrec protected final def getToArray[A](xs: Array[A], offset: Int, n: Int, elemSize: Int)(getSingle: ⇒ A)(getMult: (Array[A], Int, Int) ⇒ Unit): this.type = - if (n <= 0) this else { + @tailrec protected final def getToArray[A](xs: Array[A], offset: Int, n: Int, elemSize: Int)( + getSingle: ⇒ A + )(getMult: (Array[A], Int, Int) ⇒ Unit): this.type = + if (n <= 0) this + else { if (isEmpty) Iterator.empty.next val nDone = if (current.len >= elemSize) { val nCurrent = math.min(n, current.len / elemSize) @@ -384,7 +393,7 @@ object ByteIterator { if (!isEmpty) { val m = current.asInputStream.skip(n) normalize() - val newN = n - m + val newN = n - m val newSkipped = skipped + m if (newN > 0) skipImpl(newN, newSkipped) else newSkipped @@ -397,9 +406,8 @@ object ByteIterator { } } -/** - * An iterator over a ByteString. - */ +/** An iterator over a ByteString. + */ abstract class ByteIterator extends BufferedIterator[Byte] { def longLen: Long @@ -438,12 +446,16 @@ abstract class ByteIterator extends BufferedIterator[Byte] { // *must* be overridden by derived classes. This construction is necessary // to specialize the return type, as the method is already implemented in // the parent class. - override def takeWhile(p: Byte ⇒ Boolean): this.type = throw new UnsupportedOperationException("Method takeWhile is not implemented in ByteIterator") + override def takeWhile(p: Byte ⇒ Boolean): this.type = throw new UnsupportedOperationException( + "Method takeWhile is not implemented in ByteIterator" + ) // *must* be overridden by derived classes. This construction is necessary // to specialize the return type, as the method is already implemented in // the parent class. - override def dropWhile(p: Byte ⇒ Boolean): this.type = throw new UnsupportedOperationException("Method dropWhile is not implemented in ByteIterator") + override def dropWhile(p: Byte ⇒ Boolean): this.type = throw new UnsupportedOperationException( + "Method dropWhile is not implemented in ByteIterator" + ) override def span(p: Byte ⇒ Boolean): (ByteIterator, ByteIterator) = { val that = clone @@ -455,7 +467,8 @@ abstract class ByteIterator extends BufferedIterator[Byte] { override def indexWhere(p: Byte ⇒ Boolean): Int = { var index = 0 var found = false - while (!found && hasNext) if (p(next())) { found = true } else { index += 1 } + while (!found && hasNext) if (p(next())) { found = true } + else { index += 1 } if (found) index else -1 } @@ -482,14 +495,12 @@ abstract class ByteIterator extends BufferedIterator[Byte] { target } - /** - * Get a single Byte from this iterator. Identical to next(). - */ + /** Get a single Byte from this iterator. Identical to next(). + */ def getByte: Byte = next() - /** - * Get a single Short from this iterator. - */ + /** Get a single Short from this iterator. + */ def getShort(implicit byteOrder: ByteOrder): Short = { if (byteOrder == ByteOrder.BIG_ENDIAN) ((next() & 0xff) << 8 | (next() & 0xff) << 0).toShort @@ -498,9 +509,8 @@ abstract class ByteIterator extends BufferedIterator[Byte] { else throw new IllegalArgumentException("Unknown byte order " + byteOrder) } - /** - * Get a single Int from this iterator. - */ + /** Get a single Int from this iterator. + */ def getInt(implicit byteOrder: ByteOrder): Int = { if (byteOrder == ByteOrder.BIG_ENDIAN) ((next() & 0xff) << 24 @@ -515,9 +525,8 @@ abstract class ByteIterator extends BufferedIterator[Byte] { else throw new IllegalArgumentException("Unknown byte order " + byteOrder) } - /** - * Get a single Long from this iterator. - */ + /** Get a single Long from this iterator. + */ def getLong(implicit byteOrder: ByteOrder): Long = { if (byteOrder == ByteOrder.BIG_ENDIAN) ((next().toLong & 0xff) << 56 @@ -540,10 +549,9 @@ abstract class ByteIterator extends BufferedIterator[Byte] { else throw new IllegalArgumentException("Unknown byte order " + byteOrder) } - /** - * Get a Long from this iterator where only the least significant `n` - * bytes were encoded. - */ + /** Get a Long from this iterator where only the least significant `n` + * bytes were encoded. + */ def getLongPart(n: Int)(implicit byteOrder: ByteOrder): Long = { if (byteOrder == ByteOrder.BIG_ENDIAN) { var x = 0L @@ -562,106 +570,90 @@ abstract class ByteIterator extends BufferedIterator[Byte] { def getDouble(implicit byteOrder: ByteOrder): Double = java.lang.Double.longBitsToDouble(getLong(byteOrder)) - /** - * Get a specific number of Bytes from this iterator. In contrast to - * copyToArray, this method will fail if this.len < xs.length. - */ + /** Get a specific number of Bytes from this iterator. In contrast to + * copyToArray, this method will fail if this.len < xs.length. + */ def getBytes(xs: Array[Byte]): this.type = getBytes(xs, 0, xs.length) - /** - * Get a specific number of Bytes from this iterator. In contrast to - * copyToArray, this method will fail if length < n or if (xs.length - offset) < n. - */ + /** Get a specific number of Bytes from this iterator. In contrast to + * copyToArray, this method will fail if length < n or if (xs.length - offset) < n. + */ def getBytes(xs: Array[Byte], offset: Int, n: Int): this.type - /** - * Get a specific number of Bytes from this iterator. In contrast to - * copyToArray, this method will fail if this.len < n. - */ + /** Get a specific number of Bytes from this iterator. In contrast to + * copyToArray, this method will fail if this.len < n. + */ def getBytes(n: Int): Array[Byte] = { val bytes = new Array[Byte](n) getBytes(bytes, 0, n) bytes } - /** - * Get a ByteString with specific number of Bytes from this iterator. In contrast to - * copyToArray, this method will fail if this.len < n. - */ + /** Get a ByteString with specific number of Bytes from this iterator. In contrast to + * copyToArray, this method will fail if this.len < n. + */ def getByteString(n: Int): ByteString = { val bs = clone.take(n).toByteString drop(n) bs } - /** - * Get a number of Shorts from this iterator. - */ + /** Get a number of Shorts from this iterator. + */ def getShorts(xs: Array[Short])(implicit byteOrder: ByteOrder): this.type = getShorts(xs, 0, xs.length)(byteOrder) - /** - * Get a number of Shorts from this iterator. - */ + /** Get a number of Shorts from this iterator. + */ def getShorts(xs: Array[Short], offset: Int, n: Int)(implicit byteOrder: ByteOrder): this.type - /** - * Get a number of Ints from this iterator. - */ + /** Get a number of Ints from this iterator. + */ def getInts(xs: Array[Int])(implicit byteOrder: ByteOrder): this.type = getInts(xs, 0, xs.length)(byteOrder) - /** - * Get a number of Ints from this iterator. - */ + /** Get a number of Ints from this iterator. + */ def getInts(xs: Array[Int], offset: Int, n: Int)(implicit byteOrder: ByteOrder): this.type - /** - * Get a number of Longs from this iterator. - */ + /** Get a number of Longs from this iterator. + */ def getLongs(xs: Array[Long])(implicit byteOrder: ByteOrder): this.type = getLongs(xs, 0, xs.length)(byteOrder) - /** - * Get a number of Longs from this iterator. - */ + /** Get a number of Longs from this iterator. + */ def getLongs(xs: Array[Long], offset: Int, n: Int)(implicit byteOrder: ByteOrder): this.type - /** - * Get a number of Floats from this iterator. - */ + /** Get a number of Floats from this iterator. + */ def getFloats(xs: Array[Float])(implicit byteOrder: ByteOrder): this.type = getFloats(xs, 0, xs.length)(byteOrder) - /** - * Get a number of Floats from this iterator. - */ + /** Get a number of Floats from this iterator. + */ def getFloats(xs: Array[Float], offset: Int, n: Int)(implicit byteOrder: ByteOrder): this.type - /** - * Get a number of Doubles from this iterator. - */ + /** Get a number of Doubles from this iterator. + */ def getDoubles(xs: Array[Double])(implicit byteOrder: ByteOrder): this.type = getDoubles(xs, 0, xs.length)(byteOrder) - /** - * Get a number of Doubles from this iterator. - */ + /** Get a number of Doubles from this iterator. + */ def getDoubles(xs: Array[Double], offset: Int, n: Int)(implicit byteOrder: ByteOrder): this.type - /** - * Copy as many bytes as possible to a ByteBuffer, starting from it's - * current position. This method will not overflow the buffer. - * - * @return the number of bytes actually copied - */ + /** Copy as many bytes as possible to a ByteBuffer, starting from it's + * current position. This method will not overflow the buffer. + * + * @return the number of bytes actually copied + */ /** For performance sensitive code, call take() directly on ByteString (it's optimised there) */ def copyToBuffer(buffer: ByteBuffer): Int - /** - * Directly wraps this ByteIterator in an InputStream without copying. - * Read and skip operations on the stream will advance the iterator - * accordingly. - */ + /** Directly wraps this ByteIterator in an InputStream without copying. + * Read and skip operations on the stream will advance the iterator + * accordingly. + */ def asInputStream: java.io.InputStream } diff --git a/larray-bytestring/src/main/scala/akka/util/ByteString.scala b/larray-bytestring/src/main/scala/akka/util/ByteString.scala index c30aa800..a8750cf9 100644 --- a/larray-bytestring/src/main/scala/akka/util/ByteString.scala +++ b/larray-bytestring/src/main/scala/akka/util/ByteString.scala @@ -1,6 +1,5 @@ -/** - * Copyright (C) 2009-2017 Lightbend Inc. - */ +/** Copyright (C) 2009-2017 Lightbend Inc. + */ package akka.util @@ -21,121 +20,102 @@ import xerial.larray.LArray object ByteString { - /** - * Creates a new ByteString by copying a byte array. - */ + /** Creates a new ByteString by copying a byte array. + */ def apply(bytes: Array[Byte]): ByteString = CompactByteString(bytes) - /** - * Creates a new ByteString by copying bytes. - */ + /** Creates a new ByteString by copying bytes. + */ def apply(bytes: Byte*): ByteString = CompactByteString(bytes: _*) - /** - * Creates a new ByteString by converting from integral numbers to bytes. - */ + /** Creates a new ByteString by converting from integral numbers to bytes. + */ def apply[T](bytes: T*)(implicit num: Integral[T]): ByteString = CompactByteString(bytes: _*)(num) - /** - * Creates a new ByteString by copying bytes from a ByteBuffer. - */ + /** Creates a new ByteString by copying bytes from a ByteBuffer. + */ def apply(bytes: ByteBuffer): ByteString = CompactByteString(bytes) - /** - * Creates a new ByteString by encoding a String as UTF-8. - */ + /** Creates a new ByteString by encoding a String as UTF-8. + */ def apply(string: String): ByteString = apply(string, StandardCharsets.UTF_8) - /** - * Creates a new ByteString by encoding a String with a charset. - */ + /** Creates a new ByteString by encoding a String with a charset. + */ def apply(string: String, charset: String): ByteString = CompactByteString(string, charset) - /** - * Creates a new ByteString by encoding a String with a charset. - */ + /** Creates a new ByteString by encoding a String with a charset. + */ def apply(string: String, charset: Charset): ByteString = CompactByteString(string, charset) - /** - * Creates a new ByteString by copying a byte array. - */ + /** Creates a new ByteString by copying a byte array. + */ def fromArray(array: Array[Byte]): ByteString = apply(array) - /** - * Unsafe API: Use only in situations you are completely confident that this is what - * you need, and that you understand the implications documented below. - * - * Creates a ByteString without copying the passed in byte array, unlike other factory - * methods defined on ByteString. This method of creating a ByteString saves one array - * copy and allocation and therefore can lead to better performance, however it also means - * that one MUST NOT modify the passed in array, or unexpected immutable data structure - * contract-breaking behavior will manifest itself. - * - * This API is intended for users who have obtained an byte array from some other API, and - * want wrap it into an ByteArray, and from there on only use that reference (the ByteString) - * to operate on the wrapped data. For all other intents and purposes, please use the usual - * apply and create methods - which provide the immutability guarantees by copying the array. - * - */ + /** Unsafe API: Use only in situations you are completely confident that this is what + * you need, and that you understand the implications documented below. + * + * Creates a ByteString without copying the passed in byte array, unlike other factory + * methods defined on ByteString. This method of creating a ByteString saves one array + * copy and allocation and therefore can lead to better performance, however it also means + * that one MUST NOT modify the passed in array, or unexpected immutable data structure + * contract-breaking behavior will manifest itself. + * + * This API is intended for users who have obtained an byte array from some other API, and + * want wrap it into an ByteArray, and from there on only use that reference (the ByteString) + * to operate on the wrapped data. For all other intents and purposes, please use the usual + * apply and create methods - which provide the immutability guarantees by copying the array. + */ def fromArrayUnsafe(array: Array[Byte]): ByteString = ByteString1C(array) - /** - * Creates a new ByteString by copying length bytes starting at offset from - * an Array. - */ + /** Creates a new ByteString by copying length bytes starting at offset from + * an Array. + */ def fromArray(array: Array[Byte], offset: Int, length: Int): ByteString = CompactByteString.fromArray(array, offset, length) - /** - * Unsafe API: Use only in situations you are completely confident that this is what - * you need, and that you understand the implications documented below. - * - * Creates a ByteString without copying the passed in byte array, unlike other factory - * methods defined on ByteString. This method of creating a ByteString saves one array - * copy and allocation and therefore can lead to better performance, however it also means - * that one MUST NOT modify the passed in array, or unexpected immutable data structure - * contract-breaking behavior will manifest itself. - * - * This API is intended for users who have obtained an byte array from some other API, and - * want wrap it into an ByteArray, and from there on only use that reference (the ByteString) - * to operate on the wrapped data. For all other intents and purposes, please use the usual - * apply and create methods - which provide the immutability guarantees by copying the array. - * - */ + /** Unsafe API: Use only in situations you are completely confident that this is what + * you need, and that you understand the implications documented below. + * + * Creates a ByteString without copying the passed in byte array, unlike other factory + * methods defined on ByteString. This method of creating a ByteString saves one array + * copy and allocation and therefore can lead to better performance, however it also means + * that one MUST NOT modify the passed in array, or unexpected immutable data structure + * contract-breaking behavior will manifest itself. + * + * This API is intended for users who have obtained an byte array from some other API, and + * want wrap it into an ByteArray, and from there on only use that reference (the ByteString) + * to operate on the wrapped data. For all other intents and purposes, please use the usual + * apply and create methods - which provide the immutability guarantees by copying the array. + */ def fromArrayUnsafe(array: Array[Byte], offset: Int, length: Int): ByteString = ByteString1(array, offset, length) - /** - * JAVA API - * Creates a new ByteString by copying an int array by converting from integral numbers to bytes. - */ + /** JAVA API + * Creates a new ByteString by copying an int array by converting from integral numbers to bytes. + */ @varargs def fromInts(array: Int*): ByteString = apply(array: _*)(scala.math.Numeric.IntIsIntegral) - /** - * Creates a new ByteString which will contain the UTF-8 representation of the given String - */ + /** Creates a new ByteString which will contain the UTF-8 representation of the given String + */ def fromString(string: String): ByteString = apply(string) - /** - * Creates a new ByteString which will contain the representation of the given String in the given charset - */ + /** Creates a new ByteString which will contain the representation of the given String in the given charset + */ def fromString(string: String, charset: String): ByteString = apply(string, charset) - /** - * Creates a new ByteString which will contain the representation of the given String in the given charset - */ + /** Creates a new ByteString which will contain the representation of the given String in the given charset + */ def fromString(string: String, charset: Charset): ByteString = apply(string, charset) - /** - * Standard "UTF-8" charset - */ + /** Standard "UTF-8" charset + */ val UTF_8: String = StandardCharsets.UTF_8.name() - /** - * Creates a new ByteString by copying bytes out of a ByteBuffer. - */ + /** Creates a new ByteString by copying bytes out of a ByteBuffer. + */ def fromByteBuffer(buffer: ByteBuffer): ByteString = apply(buffer) val empty: ByteString = CompactByteString(Array.empty[Byte]) @@ -151,28 +131,27 @@ object ByteString { implicit val canBuildFrom: CanBuildFrom[TraversableOnce[Byte], Byte, ByteString] = new CanBuildFrom[TraversableOnce[Byte], Byte, ByteString] { def apply(ignore: TraversableOnce[Byte]): ByteStringBuilder = newBuilder - def apply(): ByteStringBuilder = newBuilder + def apply(): ByteStringBuilder = newBuilder } private[akka] object ByteString1C extends Companion { - def fromString(s: String): ByteString1C = apply(s.getBytes) - def apply(bytes: Array[Byte]): ByteString1C = apply(BSLArray.toLArray(bytes)) + def fromString(s: String): ByteString1C = apply(s.getBytes) + def apply(bytes: Array[Byte]): ByteString1C = apply(BSLArray.toLArray(bytes)) def apply(bytes: LArray[Byte]): ByteString1C = new ByteString1C(bytes) - val SerializationIdentity = 1.toByte + val SerializationIdentity = 1.toByte def readFromInputStream(is: ObjectInputStream): ByteString1C = { val length = is.readInt() - val arr = new Array[Byte](length) + val arr = new Array[Byte](length) is.readFully(arr, 0, length) ByteString1C(arr) } } - /** - * A compact (unsliced) and unfragmented ByteString, implementation of ByteString1C. - */ + /** A compact (unsliced) and unfragmented ByteString, implementation of ByteString1C. + */ @SerialVersionUID(3956956327691936932L) - final class ByteString1C private(private val bytes: LArray[Byte]) extends CompactByteString { + final class ByteString1C private (private val bytes: LArray[Byte]) extends CompactByteString { def apply(idx: Int): Byte = bytes(idx) override def length: Int = Math.min(bytes.length, Int.MaxValue).toInt @@ -226,7 +205,7 @@ object ByteString { if (from >= length) -1 else { var found = -1 - var i = math.max(from, 0) + var i = math.max(from, 0) while (i < length && found == -1) { if (bytes(i) == elem) found = i i += 1 @@ -249,7 +228,7 @@ object ByteString { /** INTERNAL API: Specialized for internal use, writing multiple ByteString1C into the same ByteBuffer. */ private[akka] def writeToBuffer(buffer: ByteBuffer, offset: Int): Int = { val copyLength = Math.min(buffer.remaining, offset + length) - if (copyLength > 0) for (i <- offset until (offset + copyLength)) buffer.put(bytes(i)) + if (copyLength > 0) for (i ← offset until (offset + copyLength)) buffer.put(bytes(i)) copyLength } @@ -262,8 +241,8 @@ object ByteString { /** INTERNAL API: ByteString backed by exactly one array, with start / end markers */ private[akka] object ByteString1 extends Companion { - val empty: ByteString1 = new ByteString1(LArray.emptyByteArray) - def fromString(s: String): ByteString1 = apply(s.getBytes) + val empty: ByteString1 = new ByteString1(LArray.emptyByteArray) + def fromString(s: String): ByteString1 = apply(s.getBytes) def apply(bytes: Array[Byte]): ByteString1 = apply(bytes, 0, bytes.length) def apply(bytes: Array[Byte], startIndex: Int, length: Int): ByteString1 = if (length == 0) empty @@ -279,10 +258,11 @@ object ByteString { ByteString1C.readFromInputStream(is).toByteString1 } - /** - * An unfragmented ByteString. - */ - final class ByteString1 private(private val bytes: LArray[Byte], private val startIndex: Long, val longLength: Long) extends ByteString with Serializable { + /** An unfragmented ByteString. + */ + final class ByteString1 private (private val bytes: LArray[Byte], private val startIndex: Long, val longLength: Long) + extends ByteString + with Serializable { def length: Int = longLength.toInt private def this(bytes: LArray[Byte]) = this(bytes, 0, bytes.length) @@ -302,7 +282,7 @@ object ByteString { private[akka] def writeToOutputStream(os: ObjectOutputStream): Unit = { os.writeInt(length) // TODO: Support long - for (i <- startIndex until length) os.writeByte(bytes(i)) + for (i ← startIndex until length) os.writeByte(bytes(i)) } def isCompact: Boolean = (length == bytes.length) @@ -345,7 +325,7 @@ object ByteString { private[akka] def writeToBuffer(buffer: ByteBuffer): Int = { val copyLength = Math.min(buffer.remaining, length) if (copyLength > 0) { - for (i <- startIndex until (startIndex + copyLength)) buffer.put(bytes(i)) + for (i ← startIndex until (startIndex + copyLength)) buffer.put(bytes(i)) drop(copyLength) } copyLength @@ -395,12 +375,12 @@ object ByteString { else if (this.isEmpty) that else that match { - case b: ByteString1C => ByteStrings(this, b.toByteString1) - case b: ByteString1 => + case b: ByteString1C ⇒ ByteStrings(this, b.toByteString1) + case b: ByteString1 ⇒ if ((bytes eq b.bytes) && (startIndex + length == b.startIndex)) new ByteString1(bytes, startIndex, length + b.length) else ByteStrings(this, b) - case bs: ByteStrings => ByteStrings(this, bs) + case bs: ByteStrings ⇒ ByteStrings(this, bs) } } @@ -409,7 +389,7 @@ object ByteString { if (from >= length) -1 else { var found = -1 - var i = math.max(from, 0) + var i = math.max(from, 0) while (i < length && found == -1) { if (bytes(startIndex + i) == elem) found = i i += 1 @@ -422,7 +402,7 @@ object ByteString { } private[akka] object ByteStrings extends Companion { - def apply(bytestrings: Vector[ByteString1]): ByteString = new ByteStrings(bytestrings, (0 /: bytestrings) (_ + _.length)) + def apply(bytestrings: Vector[ByteString1]): ByteString = new ByteStrings(bytestrings, (0 /: bytestrings)(_ + _.length)) def apply(bytestrings: Vector[ByteString1], length: Int): ByteString = new ByteStrings(bytestrings, length) @@ -458,7 +438,8 @@ object ByteString { def compare(b1: ByteString, b2: ByteString): Int = if (b1.isEmpty) if (b2.isEmpty) 0 else 2 - else if (b2.isEmpty) 1 else 3 + else if (b2.isEmpty) 1 + else 3 val SerializationIdentity = 2.toByte @@ -466,7 +447,7 @@ object ByteString { val nByteStrings = is.readInt() val builder = new VectorBuilder[ByteString1] - var length = 0 + var length = 0 builder.sizeHint(nByteStrings) @@ -480,16 +461,15 @@ object ByteString { } } - /** - * A ByteString with 2 or more fragments. - */ - final class ByteStrings private(private[akka] val bytestrings: Vector[ByteString1], val length: Int) extends ByteString with Serializable { + /** A ByteString with 2 or more fragments. + */ + final class ByteStrings private (private[akka] val bytestrings: Vector[ByteString1], val length: Int) extends ByteString with Serializable { if (bytestrings.isEmpty) throw new IllegalArgumentException("bytestrings must not be empty") if (bytestrings.head.isEmpty) throw new IllegalArgumentException("bytestrings.head must not be empty") def apply(idx: Int): Byte = { if (0 <= idx && idx < length) { - var pos = 0 + var pos = 0 var seen = 0 while (idx >= seen + bytestrings(pos).length) { seen += bytestrings(pos).length @@ -508,11 +488,12 @@ object ByteString { def ++(that: ByteString): ByteString = { if (that.isEmpty) this else if (this.isEmpty) that - else that match { - case b: ByteString1C ⇒ ByteStrings(this, b.toByteString1) - case b: ByteString1 ⇒ ByteStrings(this, b) - case bs: ByteStrings ⇒ ByteStrings(this, bs) - } + else + that match { + case b: ByteString1C ⇒ ByteStrings(this, b.toByteString1) + case b: ByteString1 ⇒ ByteStrings(this, b) + case bs: ByteStrings ⇒ ByteStrings(this, bs) + } } private[akka] def byteStringCompanion = ByteStrings @@ -530,7 +511,7 @@ object ByteString { def compact: CompactByteString = { if (isCompact) bytestrings.head.compact else { - val ar = new Array[Byte](length) + val ar = new Array[Byte](length) var pos = 0 bytestrings foreach { b ⇒ b.copyToArray(ar, pos, b.length) @@ -596,7 +577,8 @@ object ByteString { new ByteStrings( bytestrings.dropRight(fullDrops + 1) :+ bytestrings(byteStringsSize - fullDrops - 1) .dropRight1(remainingToDrop), - length - n) + length - n + ) } else { dropRightWithFullDropsAndRemainig(fullDrops + 1, remainingToDrop - bs.length) } @@ -685,9 +667,10 @@ object ByteString { } private[akka] object Companion { - private val companionMap = Seq(ByteString1, ByteString1C, ByteStrings). - map(x ⇒ x.SerializationIdentity → x).toMap. - withDefault(x ⇒ throw new IllegalArgumentException("Invalid serialization id " + x)) + private val companionMap = Seq(ByteString1, ByteString1C, ByteStrings) + .map(x ⇒ x.SerializationIdentity → x) + .toMap + .withDefault(x ⇒ throw new IllegalArgumentException("Invalid serialization id " + x)) def apply(from: Byte): Companion = companionMap(from) } @@ -700,14 +683,13 @@ object ByteString { } -/** - * A rope-like immutable data structure containing bytes. - * The goal of this structure is to reduce copying of arrays - * when concatenating and slicing sequences of bytes, - * and also providing a thread safe way of working with bytes. - * - * TODO: Add performance characteristics - */ +/** A rope-like immutable data structure containing bytes. + * The goal of this structure is to reduce copying of arrays + * when concatenating and slicing sequences of bytes, + * and also providing a thread safe way of working with bytes. + * + * TODO: Add performance characteristics + */ sealed abstract class ByteString extends IndexedSeq[Byte] with IndexedSeqOptimized[Byte, ByteString] { def apply(idx: Int): Byte @@ -783,11 +765,10 @@ sealed abstract class ByteString extends IndexedSeq[Byte] with IndexedSeqOptimiz super.toString } - /** - * Java API: copy this ByteString into a fresh byte array - * - * @return this ByteString copied into a byte array - */ + /** Java API: copy this ByteString into a fresh byte array + * + * @return this ByteString copied into a byte array + */ protected[ByteString] def toArray: Array[Byte] = toArray[Byte] override def toArray[B >: Byte](implicit arg0: ClassTag[B]): Array[B] = iterator.toArray @@ -799,86 +780,73 @@ sealed abstract class ByteString extends IndexedSeq[Byte] with IndexedSeqOptimiz private[akka] def writeToOutputStream(os: ObjectOutputStream): Unit - /** - * Efficiently concatenate another ByteString. - */ + /** Efficiently concatenate another ByteString. + */ def ++(that: ByteString): ByteString - /** - * Java API: efficiently concatenate another ByteString. - */ + /** Java API: efficiently concatenate another ByteString. + */ def concat(that: ByteString): ByteString = this ++ that - /** - * Copy as many bytes as possible to a ByteBuffer, starting from it's - * current position. This method will not overflow the buffer. - * - * @param buffer a ByteBuffer to copy bytes to - * @return the number of bytes actually copied - */ + /** Copy as many bytes as possible to a ByteBuffer, starting from it's + * current position. This method will not overflow the buffer. + * + * @param buffer a ByteBuffer to copy bytes to + * @return the number of bytes actually copied + */ // *must* be overridden by derived classes. def copyToBuffer(buffer: ByteBuffer): Int = - throw new UnsupportedOperationException( - s"Method copyToBuffer is not implemented in ByteString, failed for buffer $buffer") - - /** - * Create a new ByteString with all contents compacted into a single, - * full byte array. - * If isCompact returns true, compact is an O(1) operation, but - * might return a different object with an optimized implementation. - */ + throw new UnsupportedOperationException(s"Method copyToBuffer is not implemented in ByteString, failed for buffer $buffer") + + /** Create a new ByteString with all contents compacted into a single, + * full byte array. + * If isCompact returns true, compact is an O(1) operation, but + * might return a different object with an optimized implementation. + */ def compact: CompactByteString - /** - * Check whether this ByteString is compact in memory. - * If the ByteString is compact, it might, however, not be represented - * by an object that takes full advantage of that fact. Use compact to - * get such an object. - */ + /** Check whether this ByteString is compact in memory. + * If the ByteString is compact, it might, however, not be represented + * by an object that takes full advantage of that fact. Use compact to + * get such an object. + */ def isCompact: Boolean - /** - * Returns a read-only ByteBuffer that directly wraps this ByteString - * if it is not fragmented. - */ + /** Returns a read-only ByteBuffer that directly wraps this ByteString + * if it is not fragmented. + */ def asByteBuffer: ByteBuffer - /** - * Scala API: Returns an immutable Iterable of read-only ByteBuffers that directly wraps this ByteStrings - * all fragments. Will always have at least one entry. - */ + /** Scala API: Returns an immutable Iterable of read-only ByteBuffers that directly wraps this ByteStrings + * all fragments. Will always have at least one entry. + */ def asByteBuffers: immutable.Iterable[ByteBuffer] - /** - * Java API: Returns an Iterable of read-only ByteBuffers that directly wraps this ByteStrings - * all fragments. Will always have at least one entry. - */ + /** Java API: Returns an Iterable of read-only ByteBuffers that directly wraps this ByteStrings + * all fragments. Will always have at least one entry. + */ def getByteBuffers(): JIterable[ByteBuffer] = { import scala.collection.JavaConverters.asJavaIterableConverter asByteBuffers.asJava } - /** - * Creates a new ByteBuffer with a copy of all bytes contained in this - * ByteString. - */ + /** Creates a new ByteBuffer with a copy of all bytes contained in this + * ByteString. + */ def toByteBuffer: ByteBuffer = ByteBuffer.wrap(toArray) - /** - * Decodes this ByteString as a UTF-8 encoded String. - */ + /** Decodes this ByteString as a UTF-8 encoded String. + */ final def utf8String: String = decodeString(StandardCharsets.UTF_8) - /** - * Decodes this ByteString using a charset to produce a String. - * If you have a [[Charset]] instance available, use `decodeString(charset: java.nio.charset.Charset` instead. - */ + /** Decodes this ByteString using a charset to produce a String. + * If you have a [[Charset]] instance available, use `decodeString(charset: java.nio.charset.Charset` instead. + */ def decodeString(charset: String): String - /** - * Decodes this ByteString using a charset to produce a String. - * Avoids Charset.forName lookup in String internals, thus is preferable to `decodeString(charset: String)`. - */ + /** Decodes this ByteString using a charset to produce a String. + * Avoids Charset.forName lookup in String internals, thus is preferable to `decodeString(charset: String)`. + */ def decodeString(charset: Charset): String /* @@ -887,27 +855,24 @@ sealed abstract class ByteString extends IndexedSeq[Byte] with IndexedSeqOptimiz */ def decodeBase64: ByteString - /** - * Returns a ByteString which is the Base64 representation of this ByteString - */ + /** Returns a ByteString which is the Base64 representation of this ByteString + */ def encodeBase64: ByteString - /** - * map method that will automatically cast Int back into Byte. - */ - final def mapI(f: Byte => Int): ByteString = map(f.andThen(_.toByte)) + /** map method that will automatically cast Int back into Byte. + */ + final def mapI(f: Byte ⇒ Int): ByteString = map(f.andThen(_.toByte)) } object CompactByteString { - /** - * Creates a new CompactByteString by copying a byte array. - */ + + /** Creates a new CompactByteString by copying a byte array. + */ def apply(bytes: Array[Byte]): CompactByteString = if (bytes.isEmpty) empty else ByteString.ByteString1C(bytes.clone) - /** - * Creates a new CompactByteString by copying bytes. - */ + /** Creates a new CompactByteString by copying bytes. + */ def apply(bytes: Byte*): CompactByteString = { if (bytes.isEmpty) empty else { @@ -917,17 +882,15 @@ object CompactByteString { } } - /** - * Creates a new CompactByteString by converting from integral numbers to bytes. - */ + /** Creates a new CompactByteString by converting from integral numbers to bytes. + */ def apply[T](bytes: T*)(implicit num: Integral[T]): CompactByteString = { if (bytes.isEmpty) empty else ByteString.ByteString1C(bytes.map[Byte, Array[Byte]](x ⇒ num.toInt(x).toByte)(collection.breakOut)) } - /** - * Creates a new CompactByteString by copying bytes from a ByteBuffer. - */ + /** Creates a new CompactByteString by copying bytes from a ByteBuffer. + */ def apply(bytes: ByteBuffer): CompactByteString = { if (bytes.remaining < 1) empty else { @@ -937,27 +900,23 @@ object CompactByteString { } } - /** - * Creates a new CompactByteString by encoding a String as UTF-8. - */ + /** Creates a new CompactByteString by encoding a String as UTF-8. + */ def apply(string: String): CompactByteString = apply(string, StandardCharsets.UTF_8) - /** - * Creates a new CompactByteString by encoding a String with a charset. - */ + /** Creates a new CompactByteString by encoding a String with a charset. + */ def apply(string: String, charset: String): CompactByteString = if (string.isEmpty) empty else ByteString.ByteString1C(string.getBytes(charset)) - /** - * Creates a new CompactByteString by encoding a String with a charset. - */ + /** Creates a new CompactByteString by encoding a String with a charset. + */ def apply(string: String, charset: Charset): CompactByteString = if (string.isEmpty) empty else ByteString.ByteString1C(string.getBytes(charset)) - /** - * Creates a new CompactByteString by copying length bytes starting at offset from - * an Array. - */ + /** Creates a new CompactByteString by copying length bytes starting at offset from + * an Array. + */ def fromArray(array: Array[Byte], offset: Int, length: Int): CompactByteString = { val copyOffset = Math.max(offset, 0) val copyLength = Math.max(Math.min(array.length - copyOffset, length), 0) @@ -972,33 +931,31 @@ object CompactByteString { val empty: CompactByteString = ByteString.ByteString1C(Array.empty[Byte]) } -/** - * A compact ByteString. - * - * The ByteString is guarantied to be contiguous in memory and to use only - * as much memory as required for its contents. - */ +/** A compact ByteString. + * + * The ByteString is guarantied to be contiguous in memory and to use only + * as much memory as required for its contents. + */ sealed abstract class CompactByteString extends ByteString with Serializable { def isCompact: Boolean = true def compact: this.type = this } -/** - * A mutable builder for efficiently creating a [[akka.util.ByteString]]. - * - * The created ByteString is not automatically compacted. - */ +/** A mutable builder for efficiently creating a [[akka.util.ByteString]]. + * + * The created ByteString is not automatically compacted. + */ final class ByteStringBuilder extends Builder[Byte, ByteString] { builder ⇒ import ByteString.{ByteString1, ByteString1C, ByteStrings} - private var _length: Int = 0 + private var _length: Int = 0 private val _builder: VectorBuilder[ByteString1] = new VectorBuilder[ByteString1]() - private var _temp: Array[Byte] = _ - private var _tempLength: Int = 0 - private var _tempCapacity: Int = 0 + private var _temp: Array[Byte] = _ + private var _tempLength: Int = 0 + private var _tempCapacity: Int = 0 protected def fillArray(len: Int)(fill: (Array[Byte], Int) ⇒ Unit): this.type = { ensureTempSize(_tempLength + len) @@ -1009,11 +966,10 @@ final class ByteStringBuilder extends Builder[Byte, ByteString] { } @inline protected final def fillByteBuffer(len: Int, byteOrder: ByteOrder)(fill: ByteBuffer ⇒ Unit): this.type = { - fillArray(len) { - case (array, start) ⇒ - val buffer = ByteBuffer.wrap(array, start, len) - buffer.order(byteOrder) - fill(buffer) + fillArray(len) { case (array, start) ⇒ + val buffer = ByteBuffer.wrap(array, start, len) + buffer.order(byteOrder) + fill(buffer) } } @@ -1061,13 +1017,13 @@ final class ByteStringBuilder extends Builder[Byte, ByteString] { if (bytes.nonEmpty) { clearTemp() bytes match { - case b: ByteString1C => + case b: ByteString1C ⇒ _builder += b.toByteString1 _length += b.length - case b: ByteString1 => + case b: ByteString1 ⇒ _builder += b _length += b.length - case bs: ByteStrings => + case bs: ByteStrings ⇒ _builder ++= bs.bytestrings _length += bs.length } @@ -1123,19 +1079,16 @@ final class ByteStringBuilder extends Builder[Byte, ByteString] { this } - /** - * Java API: append a ByteString to this builder. - */ + /** Java API: append a ByteString to this builder. + */ def append(bs: ByteString): this.type = if (bs.isEmpty) this else this ++= bs - /** - * Add a single Byte to this builder. - */ + /** Add a single Byte to this builder. + */ def putByte(x: Byte): this.type = this += x - /** - * Add a single Short to this builder. - */ + /** Add a single Short to this builder. + */ def putShort(x: Int)(implicit byteOrder: ByteOrder): this.type = { if (byteOrder == ByteOrder.BIG_ENDIAN) { this += (x >>> 8).toByte @@ -1146,9 +1099,8 @@ final class ByteStringBuilder extends Builder[Byte, ByteString] { } else throw new IllegalArgumentException("Unknown byte order " + byteOrder) } - /** - * Add a single Int to this builder. - */ + /** Add a single Int to this builder. + */ def putInt(x: Int)(implicit byteOrder: ByteOrder): this.type = { fillArray(4) { (target, offset) ⇒ if (byteOrder == ByteOrder.BIG_ENDIAN) { @@ -1166,9 +1118,8 @@ final class ByteStringBuilder extends Builder[Byte, ByteString] { this } - /** - * Add a single Long to this builder. - */ + /** Add a single Long to this builder. + */ def putLong(x: Long)(implicit byteOrder: ByteOrder): this.type = { fillArray(8) { (target, offset) ⇒ if (byteOrder == ByteOrder.BIG_ENDIAN) { @@ -1194,9 +1145,8 @@ final class ByteStringBuilder extends Builder[Byte, ByteString] { this } - /** - * Add the `n` least significant bytes of the given Long to this builder. - */ + /** Add the `n` least significant bytes of the given Long to this builder. + */ def putLongPart(x: Long, n: Int)(implicit byteOrder: ByteOrder): this.type = { fillArray(n) { (target, offset) ⇒ if (byteOrder == ByteOrder.BIG_ENDIAN) { @@ -1208,95 +1158,81 @@ final class ByteStringBuilder extends Builder[Byte, ByteString] { } } - /** - * Add a single Float to this builder. - */ + /** Add a single Float to this builder. + */ def putFloat(x: Float)(implicit byteOrder: ByteOrder): this.type = putInt(java.lang.Float.floatToRawIntBits(x))(byteOrder) - /** - * Add a single Double to this builder. - */ + /** Add a single Double to this builder. + */ def putDouble(x: Double)(implicit byteOrder: ByteOrder): this.type = putLong(java.lang.Double.doubleToRawLongBits(x))(byteOrder) - /** - * Add a number of Bytes from an array to this builder. - */ + /** Add a number of Bytes from an array to this builder. + */ def putBytes(array: Array[Byte]): this.type = putBytes(array, 0, array.length) - /** - * Add a number of Bytes from an array to this builder. - */ + /** Add a number of Bytes from an array to this builder. + */ def putBytes(array: Array[Byte], start: Int, len: Int): this.type = fillArray(len) { case (target, targetOffset) ⇒ Array.copy(array, start, target, targetOffset, len) } - /** - * Add a number of Shorts from an array to this builder. - */ + /** Add a number of Shorts from an array to this builder. + */ def putShorts(array: Array[Short])(implicit byteOrder: ByteOrder): this.type = putShorts(array, 0, array.length)(byteOrder) - /** - * Add a number of Shorts from an array to this builder. - */ + /** Add a number of Shorts from an array to this builder. + */ def putShorts(array: Array[Short], start: Int, len: Int)(implicit byteOrder: ByteOrder): this.type = fillByteBuffer(len * 2, byteOrder) { _.asShortBuffer.put(array, start, len) } - /** - * Add a number of Ints from an array to this builder. - */ + /** Add a number of Ints from an array to this builder. + */ def putInts(array: Array[Int])(implicit byteOrder: ByteOrder): this.type = putInts(array, 0, array.length)(byteOrder) - /** - * Add a number of Ints from an array to this builder. - */ + /** Add a number of Ints from an array to this builder. + */ def putInts(array: Array[Int], start: Int, len: Int)(implicit byteOrder: ByteOrder): this.type = fillByteBuffer(len * 4, byteOrder) { _.asIntBuffer.put(array, start, len) } - /** - * Add a number of Longs from an array to this builder. - */ + /** Add a number of Longs from an array to this builder. + */ def putLongs(array: Array[Long])(implicit byteOrder: ByteOrder): this.type = putLongs(array, 0, array.length)(byteOrder) - /** - * Add a number of Longs from an array to this builder. - */ + /** Add a number of Longs from an array to this builder. + */ def putLongs(array: Array[Long], start: Int, len: Int)(implicit byteOrder: ByteOrder): this.type = fillByteBuffer(len * 8, byteOrder) { _.asLongBuffer.put(array, start, len) } - /** - * Add a number of Floats from an array to this builder. - */ + /** Add a number of Floats from an array to this builder. + */ def putFloats(array: Array[Float])(implicit byteOrder: ByteOrder): this.type = putFloats(array, 0, array.length)(byteOrder) - /** - * Add a number of Floats from an array to this builder. - */ + /** Add a number of Floats from an array to this builder. + */ def putFloats(array: Array[Float], start: Int, len: Int)(implicit byteOrder: ByteOrder): this.type = fillByteBuffer(len * 4, byteOrder) { _.asFloatBuffer.put(array, start, len) } - /** - * Add a number of Doubles from an array to this builder. - */ + /** Add a number of Doubles from an array to this builder. + */ def putDoubles(array: Array[Double])(implicit byteOrder: ByteOrder): this.type = putDoubles(array, 0, array.length)(byteOrder) - /** - * Add a number of Doubles from an array to this builder. - */ + /** Add a number of Doubles from an array to this builder. + */ def putDoubles(array: Array[Double], start: Int, len: Int)(implicit byteOrder: ByteOrder): this.type = fillByteBuffer(len * 8, byteOrder) { _.asDoubleBuffer.put(array, start, len) @@ -1319,10 +1255,9 @@ final class ByteStringBuilder extends Builder[Byte, ByteString] { ByteStrings(bytestrings, _length) } - /** - * Directly wraps this ByteStringBuilder in an OutputStream. Write - * operations on the stream are forwarded to the builder. - */ + /** Directly wraps this ByteStringBuilder in an OutputStream. Write + * operations on the stream are forwarded to the builder. + */ def asOutputStream: java.io.OutputStream = new java.io.OutputStream { def write(b: Int): Unit = builder += b.toByte @@ -1331,13 +1266,11 @@ final class ByteStringBuilder extends Builder[Byte, ByteString] { } } - /** - * Tests whether this ByteStringBuilder is empty. - */ + /** Tests whether this ByteStringBuilder is empty. + */ def isEmpty: Boolean = _length == 0 - /** - * Tests whether this ByteStringBuilder is not empty. - */ + /** Tests whether this ByteStringBuilder is not empty. + */ def nonEmpty: Boolean = _length > 0 } diff --git a/metadata/imageio/src/main/scala/com/karasiq/shadowcloud/metadata/imageio/ImageIOMetadataProvider.scala b/metadata/imageio/src/main/scala/com/karasiq/shadowcloud/metadata/imageio/ImageIOMetadataProvider.scala index d93c89e9..551d17ef 100644 --- a/metadata/imageio/src/main/scala/com/karasiq/shadowcloud/metadata/imageio/ImageIOMetadataProvider.scala +++ b/metadata/imageio/src/main/scala/com/karasiq/shadowcloud/metadata/imageio/ImageIOMetadataProvider.scala @@ -6,7 +6,7 @@ import com.karasiq.shadowcloud.metadata.{MetadataParser, MetadataProvider, MimeD class ImageIOMetadataProvider(rootConfig: Config) extends MetadataProvider { protected object imageioConfig { - val config = rootConfig.getConfig("metadata.imageio") + val config = rootConfig.getConfig("metadata.imageio") val thumbnailsConfig = config.getConfig("thumbnails") } diff --git a/metadata/imageio/src/main/scala/com/karasiq/shadowcloud/metadata/imageio/ImageIOThumbnailCreator.scala b/metadata/imageio/src/main/scala/com/karasiq/shadowcloud/metadata/imageio/ImageIOThumbnailCreator.scala index d723a44a..e18b91f4 100644 --- a/metadata/imageio/src/main/scala/com/karasiq/shadowcloud/metadata/imageio/ImageIOThumbnailCreator.scala +++ b/metadata/imageio/src/main/scala/com/karasiq/shadowcloud/metadata/imageio/ImageIOThumbnailCreator.scala @@ -30,10 +30,10 @@ private[imageio] class ImageIOThumbnailCreator(config: Config) extends MetadataP protected object thumbnailSettings extends ConfigImplicits { val parserConfig = MetadataParserConfig(config) - val sizeLimit = config.getBytesInt("size-limit") + val sizeLimit = config.getBytesInt("size-limit") - val size = config.getInt("size") - val format = config.getString("format") + val size = config.getInt("size") + val format = config.getString("format") val quality = config.getInt("quality") } @@ -49,24 +49,28 @@ private[imageio] class ImageIOThumbnailCreator(config: Config) extends MetadataP }) val createImageData = builder.add(Flow[BufferedImage].map { image ⇒ - Metadata(Some(Metadata.Tag(PluginId, ParserId, Disposition.METADATA)), - Metadata.Value.ImageData(Metadata.ImageData(image.getWidth, image.getHeight))) + Metadata( + Some(Metadata.Tag(PluginId, ParserId, Disposition.METADATA)), + Metadata.Value.ImageData(Metadata.ImageData(image.getWidth, image.getHeight)) + ) }) val resizeImage = builder.add(Flow[BufferedImage].map { originalImage ⇒ - val image = ImageIOResizer.resize(originalImage, thumbnailSettings.size) + val image = ImageIOResizer.resize(originalImage, thumbnailSettings.size) val outputStream = ByteStringOutputStream() ImageIOResizer.compress(image, outputStream, thumbnailSettings.format, thumbnailSettings.quality) outputStream.toByteString }) val createPreview = builder.add(Flow[ByteString].map { data ⇒ - Metadata(Some(Metadata.Tag(PluginId, ParserId, Disposition.PREVIEW)), - Metadata.Value.Thumbnail(Metadata.Thumbnail(thumbnailSettings.format, data))) + Metadata( + Some(Metadata.Tag(PluginId, ParserId, Disposition.PREVIEW)), + Metadata.Value.Thumbnail(Metadata.Thumbnail(thumbnailSettings.format, data)) + ) }) - val readImage = builder.add(Flow[ByteString].fold(ByteString.empty)(_ ++ _)) - val processImage = builder.add(Broadcast[BufferedImage](2)) + val readImage = builder.add(Flow[ByteString].fold(ByteString.empty)(_ ++ _)) + val processImage = builder.add(Broadcast[BufferedImage](2)) val writeMetadata = builder.add(Concat[Metadata](2)) readImage ~> loadImage ~> processImage diff --git a/metadata/imageio/src/main/scala/com/karasiq/shadowcloud/metadata/imageio/utils/ImageIOResizer.scala b/metadata/imageio/src/main/scala/com/karasiq/shadowcloud/metadata/imageio/utils/ImageIOResizer.scala index 8bf82931..c6ec2e15 100644 --- a/metadata/imageio/src/main/scala/com/karasiq/shadowcloud/metadata/imageio/utils/ImageIOResizer.scala +++ b/metadata/imageio/src/main/scala/com/karasiq/shadowcloud/metadata/imageio/utils/ImageIOResizer.scala @@ -18,38 +18,38 @@ object ImageIOResizer { object Quality { import RenderingHints._ - + def apply(values: (RenderingHints.Key, AnyRef)*): Quality = { Quality(values.toMap) } val fast = Quality( - KEY_RENDERING → VALUE_RENDER_SPEED, - KEY_COLOR_RENDERING → VALUE_COLOR_RENDER_SPEED, - KEY_ANTIALIASING → VALUE_ANTIALIAS_OFF, + KEY_RENDERING → VALUE_RENDER_SPEED, + KEY_COLOR_RENDERING → VALUE_COLOR_RENDER_SPEED, + KEY_ANTIALIASING → VALUE_ANTIALIAS_OFF, KEY_ALPHA_INTERPOLATION → VALUE_ALPHA_INTERPOLATION_SPEED, - KEY_INTERPOLATION → VALUE_INTERPOLATION_NEAREST_NEIGHBOR + KEY_INTERPOLATION → VALUE_INTERPOLATION_NEAREST_NEIGHBOR ) val default = Quality( - KEY_RENDERING → VALUE_RENDER_SPEED, - KEY_COLOR_RENDERING → VALUE_COLOR_RENDER_SPEED, - KEY_ANTIALIASING → VALUE_ANTIALIAS_ON, + KEY_RENDERING → VALUE_RENDER_SPEED, + KEY_COLOR_RENDERING → VALUE_COLOR_RENDER_SPEED, + KEY_ANTIALIASING → VALUE_ANTIALIAS_ON, KEY_ALPHA_INTERPOLATION → VALUE_ALPHA_INTERPOLATION_SPEED, - KEY_INTERPOLATION → VALUE_INTERPOLATION_BILINEAR + KEY_INTERPOLATION → VALUE_INTERPOLATION_BILINEAR ) val highQuality = Quality( - KEY_RENDERING → VALUE_RENDER_QUALITY, - KEY_COLOR_RENDERING → VALUE_COLOR_RENDER_QUALITY, - KEY_ANTIALIASING → VALUE_ANTIALIAS_ON, + KEY_RENDERING → VALUE_RENDER_QUALITY, + KEY_COLOR_RENDERING → VALUE_COLOR_RENDER_QUALITY, + KEY_ANTIALIASING → VALUE_ANTIALIAS_ON, KEY_ALPHA_INTERPOLATION → VALUE_ALPHA_INTERPOLATION_QUALITY, - KEY_INTERPOLATION → VALUE_INTERPOLATION_BICUBIC + KEY_INTERPOLATION → VALUE_INTERPOLATION_BICUBIC ) } private[this] def getScaledDimension(imgSize: Dimension, boundary: Dimension): Dimension = { - var newWidth: Int = imgSize.width + var newWidth: Int = imgSize.width var newHeight: Int = imgSize.height if (imgSize.width > boundary.width) { newWidth = boundary.width @@ -64,7 +64,7 @@ object ImageIOResizer { private[this] def redrawImage(image: Image, size: Dimension, quality: Quality): BufferedImage = { val outputImage = new BufferedImage(size.width, size.height, BufferedImage.TYPE_INT_RGB) - val graphics = outputImage.createGraphics() + val graphics = outputImage.createGraphics() try { // Max quality settings graphics.setRenderingHints(quality.toJavaMap) @@ -79,7 +79,7 @@ object ImageIOResizer { def loadImage(bytes: Array[Byte]): BufferedImage = { val image = Toolkit.getDefaultToolkit.createImage(bytes) - val rgbMasks: Array[Int] = Array(0xFF0000, 0xFF00, 0xFF) + val rgbMasks: Array[Int] = Array(0xff0000, 0xff00, 0xff) val rgbOpaqueModel: ColorModel = new DirectColorModel(32, rgbMasks(0), rgbMasks(1), rgbMasks(2)) val pg = new PixelGrabber(image, 0, 0, -1, -1, true) @@ -96,11 +96,10 @@ object ImageIOResizer { } } - def compress(image: BufferedImage, outputStream: OutputStream, - format: String = "jpeg", saveQuality: Int = 80): Unit = { + def compress(image: BufferedImage, outputStream: OutputStream, format: String = "jpeg", saveQuality: Int = 80): Unit = { val imageOutputStream = ImageIO.createImageOutputStream(outputStream) try { - val writer: ImageWriter = ImageIO.getImageWritersByFormatName(format).next + val writer: ImageWriter = ImageIO.getImageWritersByFormatName(format).next val iwp: ImageWriteParam = writer.getDefaultWriteParam if (iwp.canWriteCompressed) { iwp.setCompressionMode(ImageWriteParam.MODE_EXPLICIT) @@ -116,9 +115,9 @@ object ImageIOResizer { } def resize(image: BufferedImage, size: Int, quality: Quality = Quality.default): BufferedImage = { - val imgSize = new Dimension(image.getWidth, image.getHeight) + val imgSize = new Dimension(image.getWidth, image.getHeight) val boundary = new Dimension(size, size) - val newSize = getScaledDimension(imgSize, boundary) + val newSize = getScaledDimension(imgSize, boundary) redrawImage(image, newSize, quality) } } diff --git a/metadata/imageio/src/test/scala/com/karasiq/shadowcloud/metadata/imageio/test/ImageIOMetadataProviderTest.scala b/metadata/imageio/src/test/scala/com/karasiq/shadowcloud/metadata/imageio/test/ImageIOMetadataProviderTest.scala index 0c41557e..14d8dac9 100644 --- a/metadata/imageio/src/test/scala/com/karasiq/shadowcloud/metadata/imageio/test/ImageIOMetadataProviderTest.scala +++ b/metadata/imageio/src/test/scala/com/karasiq/shadowcloud/metadata/imageio/test/ImageIOMetadataProviderTest.scala @@ -38,12 +38,12 @@ class ImageIOMetadataProviderTest extends ActorSpec with ActorSpecImplicits with .via(thumbnailCreator.parseMetadata(name, s"image/$imageType")) .runWith(TestSink.probe) - val imageData = testStream.requestNext(5 seconds) + val imageData = testStream.requestNext(5 seconds) val imageDataValue = imageData.value.imageData.get imageDataValue.width shouldBe width imageDataValue.height shouldBe height - val thumb = testStream.requestNext(5 seconds) + val thumb = testStream.requestNext(5 seconds) val thumbImage = extractThumbnail(thumb) thumbImage.getWidth shouldBe tWidth thumbImage.getHeight shouldBe tHeight @@ -57,7 +57,8 @@ class ImageIOMetadataProviderTest extends ActorSpec with ActorSpecImplicits with } private[this] def createResourceStream(name: String): Source[ByteString, NotUsed] = { - StreamConverters.fromInputStream(() ⇒ getClass.getClassLoader.getResourceAsStream(name)) + StreamConverters + .fromInputStream(() ⇒ getClass.getClassLoader.getResourceAsStream(name)) .mapMaterializedValue(_ ⇒ NotUsed) } diff --git a/metadata/javacv/src/main/scala/com/karasiq/shadowcloud/metadata/javacv/FFMPEGThumbnailCreator.scala b/metadata/javacv/src/main/scala/com/karasiq/shadowcloud/metadata/javacv/FFMPEGThumbnailCreator.scala index 92059170..45460bbe 100644 --- a/metadata/javacv/src/main/scala/com/karasiq/shadowcloud/metadata/javacv/FFMPEGThumbnailCreator.scala +++ b/metadata/javacv/src/main/scala/com/karasiq/shadowcloud/metadata/javacv/FFMPEGThumbnailCreator.scala @@ -19,8 +19,8 @@ private[javacv] object FFMPEGThumbnailCreator { private[javacv] class FFMPEGThumbnailCreator(config: Config) extends BlockingMetadataParser { protected object settings { - val parserConfig = MetadataParserConfig(config) - val thumbnailSize = config.getInt("thumbnail-size") + val parserConfig = MetadataParserConfig(config) + val thumbnailSize = config.getInt("thumbnail-size") val thumbnailQuality = config.getInt("thumbnail-quality") } @@ -35,16 +35,19 @@ private[javacv] class FFMPEGThumbnailCreator(config: Config) extends BlockingMet grabber.start() val converter = new OpenCVFrameConverter.ToIplImage() - val image = converter.convert(grabber.grabImage()) - val thumb = OpenCVThumbnailCreator.resizeIplImage(image, settings.thumbnailSize, settings.thumbnailSize) - val result = try { - val jpegBytes = ByteString.fromArrayUnsafe(JavaCV.asJpeg(thumb, settings.thumbnailQuality)) - Metadata(Some(Metadata.Tag("javacv", "ffmpeg", Metadata.Tag.Disposition.PREVIEW)), - Metadata.Value.Thumbnail(Metadata.Thumbnail("jpeg", jpegBytes))) - } finally { - image.release() - thumb.release() - } + val image = converter.convert(grabber.grabImage()) + val thumb = OpenCVThumbnailCreator.resizeIplImage(image, settings.thumbnailSize, settings.thumbnailSize) + val result = + try { + val jpegBytes = ByteString.fromArrayUnsafe(JavaCV.asJpeg(thumb, settings.thumbnailQuality)) + Metadata( + Some(Metadata.Tag("javacv", "ffmpeg", Metadata.Tag.Disposition.PREVIEW)), + Metadata.Value.Thumbnail(Metadata.Thumbnail("jpeg", jpegBytes)) + ) + } finally { + image.release() + thumb.release() + } Source.single(result) } finally grabber.stop() diff --git a/metadata/javacv/src/main/scala/com/karasiq/shadowcloud/metadata/javacv/JavaCV.scala b/metadata/javacv/src/main/scala/com/karasiq/shadowcloud/metadata/javacv/JavaCV.scala index d377a705..3ad8d08a 100644 --- a/metadata/javacv/src/main/scala/com/karasiq/shadowcloud/metadata/javacv/JavaCV.scala +++ b/metadata/javacv/src/main/scala/com/karasiq/shadowcloud/metadata/javacv/JavaCV.scala @@ -3,12 +3,11 @@ package com.karasiq.shadowcloud.metadata.javacv import org.bytedeco.javacpp.opencv_core._ import org.bytedeco.javacpp.opencv_imgcodecs._ -/** - * JavaCV helper object +/** JavaCV helper object */ private[javacv] object JavaCV { - /** - * Creates JPEG-encoded image + + /** Creates JPEG-encoded image * @param image JavaCV image * @param quality JPEG quality * @return Bytes of JPEG-encoded image @@ -16,11 +15,11 @@ private[javacv] object JavaCV { def asJpeg(image: IplImage, quality: Int = 85): Array[Byte] = { val matrix = cvEncodeImage(".jpeg", image.asCvMat(), Array(CV_IMWRITE_JPEG_QUALITY, quality, 0)) try { - val dataPtr = matrix.data_ptr() + val dataPtr = matrix.data_ptr() val dataSize = matrix.size() val outArray = new Array[Byte](dataSize) dataPtr.get(outArray, 0, dataSize) outArray } finally matrix.release() } -} \ No newline at end of file +} diff --git a/metadata/javacv/src/main/scala/com/karasiq/shadowcloud/metadata/javacv/JavaCVMetadataProvider.scala b/metadata/javacv/src/main/scala/com/karasiq/shadowcloud/metadata/javacv/JavaCVMetadataProvider.scala index a9fb6343..db36322f 100644 --- a/metadata/javacv/src/main/scala/com/karasiq/shadowcloud/metadata/javacv/JavaCVMetadataProvider.scala +++ b/metadata/javacv/src/main/scala/com/karasiq/shadowcloud/metadata/javacv/JavaCVMetadataProvider.scala @@ -8,5 +8,5 @@ private[javacv] class JavaCVMetadataProvider(rootConfig: Config) extends Metadat private[this] val config = rootConfig.getConfig("metadata.javacv") val detectors = Nil - val parsers = Vector(FFMPEGThumbnailCreator(config.getConfig("ffmpeg")), OpenCVThumbnailCreator(config.getConfig("opencv"))) + val parsers = Vector(FFMPEGThumbnailCreator(config.getConfig("ffmpeg")), OpenCVThumbnailCreator(config.getConfig("opencv"))) } diff --git a/metadata/javacv/src/main/scala/com/karasiq/shadowcloud/metadata/javacv/OpenCVThumbnailCreator.scala b/metadata/javacv/src/main/scala/com/karasiq/shadowcloud/metadata/javacv/OpenCVThumbnailCreator.scala index 4672b27b..24ede572 100644 --- a/metadata/javacv/src/main/scala/com/karasiq/shadowcloud/metadata/javacv/OpenCVThumbnailCreator.scala +++ b/metadata/javacv/src/main/scala/com/karasiq/shadowcloud/metadata/javacv/OpenCVThumbnailCreator.scala @@ -17,7 +17,7 @@ private[javacv] object OpenCVThumbnailCreator { } def getScaledDimension(width: Int, height: Int, widthConstraint: Int, heightConstraint: Int): (Int, Int) = { - var newWidth: Int = width + var newWidth: Int = width var newHeight: Int = height if (width > widthConstraint) { newWidth = widthConstraint @@ -32,7 +32,7 @@ private[javacv] object OpenCVThumbnailCreator { def resizeIplImage(image: IplImage, widthConstraint: Int, heightConstraint: Int): IplImage = { val (newWidth, newHeight) = getScaledDimension(image.width(), image.height(), widthConstraint, heightConstraint) - val thumb = cvCreateImage(cvSize(newWidth, newHeight), image.depth(), image.nChannels()) + val thumb = cvCreateImage(cvSize(newWidth, newHeight), image.depth(), image.nChannels()) cvResize(image, thumb) thumb } @@ -40,9 +40,9 @@ private[javacv] object OpenCVThumbnailCreator { private[javacv] class OpenCVThumbnailCreator(config: Config) extends MetadataParser { protected object settings { - val parserConfig = MetadataParserConfig(config) - val sizeLimit = config.getBytes("size-limit") - val thumbnailSize = config.getInt("thumbnail-size") + val parserConfig = MetadataParserConfig(config) + val sizeLimit = config.getBytes("size-limit") + val thumbnailSize = config.getInt("thumbnail-size") val thumbnailQuality = config.getInt("thumbnail-quality") } @@ -56,13 +56,16 @@ private[javacv] class OpenCVThumbnailCreator(config: Config) extends MetadataPar .via(ByteStreams.concat) .map { bytes ⇒ require(bytes.nonEmpty, "Image is empty") - val image = new IplImage(imdecode(new Mat(bytes:_*), CV_LOAD_IMAGE_UNCHANGED)) // cvDecodeImage(new CvMat(new Mat(bytes:_*)), CV_LOAD_IMAGE_UNCHANGED) + val image = + new IplImage(imdecode(new Mat(bytes: _*), CV_LOAD_IMAGE_UNCHANGED)) // cvDecodeImage(new CvMat(new Mat(bytes:_*)), CV_LOAD_IMAGE_UNCHANGED) try { val thumb = OpenCVThumbnailCreator.resizeIplImage(image, settings.thumbnailSize, settings.thumbnailSize) try { val jpegBytes = ByteString.fromArrayUnsafe(JavaCV.asJpeg(thumb, settings.thumbnailQuality)) - Metadata(Some(Metadata.Tag("javacv", "opencv", Metadata.Tag.Disposition.PREVIEW)), - Metadata.Value.Thumbnail(Metadata.Thumbnail("jpeg", jpegBytes))) + Metadata( + Some(Metadata.Tag("javacv", "opencv", Metadata.Tag.Disposition.PREVIEW)), + Metadata.Value.Thumbnail(Metadata.Thumbnail("jpeg", jpegBytes)) + ) } finally thumb.release() } finally image.release() } diff --git a/metadata/markdown/src/main/scala/com/karasiq/shadowcloud/metadata/markdown/FlexmarkMetadataParser.scala b/metadata/markdown/src/main/scala/com/karasiq/shadowcloud/metadata/markdown/FlexmarkMetadataParser.scala index a833b003..e8db1d8b 100644 --- a/metadata/markdown/src/main/scala/com/karasiq/shadowcloud/metadata/markdown/FlexmarkMetadataParser.scala +++ b/metadata/markdown/src/main/scala/com/karasiq/shadowcloud/metadata/markdown/FlexmarkMetadataParser.scala @@ -30,12 +30,21 @@ private[markdown] class FlexmarkMetadataParser(config: Config) extends MetadataP private[this] val options = { val options = new MutableDataSet - options.set(Parser.EXTENSIONS, util.Arrays.asList(TablesExtension.create(), StrikethroughExtension.create(), TaskListExtension.create(), WikiLinkExtension.create(), AutolinkExtension.create())) + options.set( + Parser.EXTENSIONS, + util.Arrays.asList( + TablesExtension.create(), + StrikethroughExtension.create(), + TaskListExtension.create(), + WikiLinkExtension.create(), + AutolinkExtension.create() + ) + ) options.set(HtmlRenderer.SOFT_BREAK, "
\n") options } - private[this] val parser = Parser.builder(options).build + private[this] val parser = Parser.builder(options).build private[this] val renderer = HtmlRenderer.builder(options).build def canParse(name: String, mime: String) = { @@ -48,7 +57,7 @@ private[markdown] class FlexmarkMetadataParser(config: Config) extends MetadataP .filter(_.nonEmpty) .map { bytes ⇒ val markdownStr = bytes.utf8String - val html = renderer.render(parser.parse(markdownStr)) + val html = renderer.render(parser.parse(markdownStr)) Metadata(Some(Metadata.Tag("markdown", "flexmark", Metadata.Tag.Disposition.PREVIEW)), Metadata.Value.Text(Metadata.Text("text/html", html))) } .named("flexmarkParse") diff --git a/metadata/markdown/src/main/scala/com/karasiq/shadowcloud/metadata/markdown/PlaintextMetadataParser.scala b/metadata/markdown/src/main/scala/com/karasiq/shadowcloud/metadata/markdown/PlaintextMetadataParser.scala index 471e0e3c..ff1299c9 100644 --- a/metadata/markdown/src/main/scala/com/karasiq/shadowcloud/metadata/markdown/PlaintextMetadataParser.scala +++ b/metadata/markdown/src/main/scala/com/karasiq/shadowcloud/metadata/markdown/PlaintextMetadataParser.scala @@ -21,7 +21,7 @@ object PlaintextMetadataParser { class PlaintextMetadataParser(config: Config) extends MetadataParser { protected object settings { val parserConfig = MetadataParserConfig(config) - val sizeLimit = config.getBytesInt("size-limit") + val sizeLimit = config.getBytesInt("size-limit") } def canParse(name: String, mime: String): Boolean = { diff --git a/metadata/markdown/src/test/scala/com/karasiq/shadowcloud/metadata/markdown/test/MarkdownMetadataProviderTest.scala b/metadata/markdown/src/test/scala/com/karasiq/shadowcloud/metadata/markdown/test/MarkdownMetadataProviderTest.scala index 12f8f8ec..ce605f09 100644 --- a/metadata/markdown/src/test/scala/com/karasiq/shadowcloud/metadata/markdown/test/MarkdownMetadataProviderTest.scala +++ b/metadata/markdown/src/test/scala/com/karasiq/shadowcloud/metadata/markdown/test/MarkdownMetadataProviderTest.scala @@ -24,7 +24,8 @@ class MarkdownMetadataProviderTest extends ActorSpec with ActorSpecImplicits wit private[this] def testParser(parser: MetadataParser)(name: String, mime: String, data: String)(doTest: (String, String) ⇒ Unit): Unit = { parser.canParse(name, mime) shouldBe true - val result = Source.single(ByteString(data)) + val result = Source + .single(ByteString(data)) .via(parser.parseMetadata(name, mime)) .runWith(Sink.seq) .futureValue diff --git a/metadata/parent/src/main/scala/com/karasiq/shadowcloud/metadata/config/MetadataParserConfig.scala b/metadata/parent/src/main/scala/com/karasiq/shadowcloud/metadata/config/MetadataParserConfig.scala index 97b293ac..07e64ef0 100644 --- a/metadata/parent/src/main/scala/com/karasiq/shadowcloud/metadata/config/MetadataParserConfig.scala +++ b/metadata/parent/src/main/scala/com/karasiq/shadowcloud/metadata/config/MetadataParserConfig.scala @@ -21,4 +21,4 @@ object MetadataParserConfig extends WrappedConfigFactory[MetadataParserConfig] { config.withDefault(Set.empty, _.getStringSet("mimes")) ) } -} \ No newline at end of file +} diff --git a/metadata/parent/src/main/scala/com/karasiq/shadowcloud/metadata/utils/BlockingMetadataParser.scala b/metadata/parent/src/main/scala/com/karasiq/shadowcloud/metadata/utils/BlockingMetadataParser.scala index 2e8d8d63..b1184c15 100644 --- a/metadata/parent/src/main/scala/com/karasiq/shadowcloud/metadata/utils/BlockingMetadataParser.scala +++ b/metadata/parent/src/main/scala/com/karasiq/shadowcloud/metadata/utils/BlockingMetadataParser.scala @@ -12,8 +12,7 @@ import com.karasiq.shadowcloud.metadata.{Metadata, MetadataParser} import scala.concurrent.duration._ import scala.util.Try -/** - * Blocking InputStream API wrapper +/** Blocking InputStream API wrapper */ trait BlockingMetadataParser extends MetadataParser { protected def parseMetadata(name: String, mime: String, inputStream: InputStream): Source[Metadata, NotUsed] diff --git a/metadata/tika/src/main/scala/com/karasiq/shadowcloud/metadata/tika/TikaAutoParser.scala b/metadata/tika/src/main/scala/com/karasiq/shadowcloud/metadata/tika/TikaAutoParser.scala index e3181514..74456d0d 100644 --- a/metadata/tika/src/main/scala/com/karasiq/shadowcloud/metadata/tika/TikaAutoParser.scala +++ b/metadata/tika/src/main/scala/com/karasiq/shadowcloud/metadata/tika/TikaAutoParser.scala @@ -26,11 +26,10 @@ private[tika] object TikaAutoParser { } } -/** - * Parses file with default Tika parser - * +/** Parses file with default Tika parser + * * @param config Parser config - */ + */ //noinspection ScalaDeprecation private[tika] final class TikaAutoParser(tika: Tika, val config: Config) extends TikaMetadataParser { // ----------------------------------------------------------------------- @@ -108,7 +107,7 @@ private[tika] final class TikaAutoParser(tika: Tika, val config: Config) extends parser.parse(inputStream, null, metadata, new ParseContext) } catch { case NonFatal(_) ⇒ - // Ignore + // Ignore } (parser.getMetadata.asScala.toVector, handlers.toVector) diff --git a/metadata/tika/src/main/scala/com/karasiq/shadowcloud/metadata/tika/TikaMetadataParser.scala b/metadata/tika/src/main/scala/com/karasiq/shadowcloud/metadata/tika/TikaMetadataParser.scala index bb79b3e4..304b2736 100644 --- a/metadata/tika/src/main/scala/com/karasiq/shadowcloud/metadata/tika/TikaMetadataParser.scala +++ b/metadata/tika/src/main/scala/com/karasiq/shadowcloud/metadata/tika/TikaMetadataParser.scala @@ -14,8 +14,7 @@ import org.apache.tika.parser.{ParseContext, Parser} import scala.collection.JavaConverters._ -/** - * Abstract metadata parser +/** Abstract metadata parser */ private[tika] trait TikaMetadataParser extends BlockingMetadataParser { val parser: Parser diff --git a/metadata/tika/src/main/scala/com/karasiq/shadowcloud/metadata/tika/TikaMetadataProvider.scala b/metadata/tika/src/main/scala/com/karasiq/shadowcloud/metadata/tika/TikaMetadataProvider.scala index a709f2fb..849ff511 100644 --- a/metadata/tika/src/main/scala/com/karasiq/shadowcloud/metadata/tika/TikaMetadataProvider.scala +++ b/metadata/tika/src/main/scala/com/karasiq/shadowcloud/metadata/tika/TikaMetadataProvider.scala @@ -9,8 +9,8 @@ import com.karasiq.shadowcloud.metadata.MetadataProvider class TikaMetadataProvider(rootConfig: Config) extends MetadataProvider { protected object tikaConfig extends ConfigImplicits { - val config = rootConfig.getConfig("metadata.tika") - val xmlConfigFile = config.optional(_.getString("xml-config-file")) + val config = rootConfig.getConfig("metadata.tika") + val xmlConfigFile = config.optional(_.getString("xml-config-file")) val autoParserConfig = config.getConfig("auto-parser") } diff --git a/metadata/tika/src/main/scala/com/karasiq/shadowcloud/metadata/tika/utils/TikaAttributes.scala b/metadata/tika/src/main/scala/com/karasiq/shadowcloud/metadata/tika/utils/TikaAttributes.scala index f4aeeedb..613668c4 100644 --- a/metadata/tika/src/main/scala/com/karasiq/shadowcloud/metadata/tika/utils/TikaAttributes.scala +++ b/metadata/tika/src/main/scala/com/karasiq/shadowcloud/metadata/tika/utils/TikaAttributes.scala @@ -5,13 +5,13 @@ import com.karasiq.shadowcloud.metadata.tika.TikaMetadata private[tika] object TikaAttributes { val ResourceName = "resourceName" - val Size = "Content-Length" + val Size = "Content-Length" val LastModified = "Last-Modified" - val ImageWidth = "tiff:ImageWidth" + val ImageWidth = "tiff:ImageWidth" val ImageHeight = "tiff:ImageLength" - val Title = "dc:title" + val Title = "dc:title" val Description = "dc:description" def optional(md: TikaMetadata, name: String): Option[String] = { diff --git a/metadata/tika/src/main/scala/com/karasiq/shadowcloud/metadata/tika/utils/TikaConversions.scala b/metadata/tika/src/main/scala/com/karasiq/shadowcloud/metadata/tika/utils/TikaConversions.scala index b2c91ae3..1bf0bdd3 100644 --- a/metadata/tika/src/main/scala/com/karasiq/shadowcloud/metadata/tika/utils/TikaConversions.scala +++ b/metadata/tika/src/main/scala/com/karasiq/shadowcloud/metadata/tika/utils/TikaConversions.scala @@ -17,7 +17,8 @@ private[tika] class TikaConversions(plugin: String, parser: String) { Option(metadata) .filter(_.size() > 0) .map { metadataMap ⇒ - val values = metadataMap.names() + val values = metadataMap + .names() .map(name ⇒ (name, Metadata.Table.Values(metadataMap.getValues(name).toVector))) .toMap Metadata(createTag(MDDisposition.METADATA), Metadata.Value.Table(Metadata.Table(values))) @@ -33,7 +34,8 @@ private[tika] class TikaConversions(plugin: String, parser: String) { Metadata.EmbeddedResources.EmbeddedResource(resourcePath.nodes, metadatas) } - val groupedResources = resources.filter(_.metadata.nonEmpty) + val groupedResources = resources + .filter(_.metadata.nonEmpty) .groupBy(_.path) .map { case (path, resources) ⇒ val allMetadatas = resources.flatMap(_.metadata) @@ -42,13 +44,12 @@ private[tika] class TikaConversions(plugin: String, parser: String) { Some(groupedResources) .filter(_.nonEmpty) - .map(resources ⇒ Metadata(createTag(MDDisposition.CONTENT), - Metadata.Value.EmbeddedResources(Metadata.EmbeddedResources(resources.toVector)))) + .map(resources ⇒ Metadata(createTag(MDDisposition.CONTENT), Metadata.Value.EmbeddedResources(Metadata.EmbeddedResources(resources.toVector)))) } def toArchiveTables(subMetadatas: Seq[Metadata], previewSize: Int): Seq[Metadata] = { val archiveFiles = for { - md ← subMetadatas if md.value.isTable + md ← subMetadatas if md.value.isTable path ← TikaAttributes.optional(md, TikaAttributes.ResourceName).map(ps ⇒ Path.fromString(ps).nodes) size ← TikaAttributes.optional(md, TikaAttributes.Size).map(_.toLong) timestamp = TikaAttributes.optional(md, TikaAttributes.LastModified).fold(0L)(TikaUtils.parseTimeString) @@ -58,11 +59,9 @@ private[tika] class TikaConversions(plugin: String, parser: String) { .filter(_.nonEmpty) .toSeq .flatMap { files ⇒ - val preview = Metadata(createTag(MDDisposition.PREVIEW), - Metadata.Value.FileList(Metadata.FileList(files.take(previewSize)))) + val preview = Metadata(createTag(MDDisposition.PREVIEW), Metadata.Value.FileList(Metadata.FileList(files.take(previewSize)))) if (files.length > previewSize) { - val full = Metadata(createTag(MDDisposition.CONTENT), - Metadata.Value.FileList(Metadata.FileList(files))) + val full = Metadata(createTag(MDDisposition.CONTENT), Metadata.Value.FileList(Metadata.FileList(files))) Seq(preview, full) } else { Seq(preview) @@ -86,16 +85,15 @@ private[tika] class TikaConversions(plugin: String, parser: String) { def toImageData(metadataTable: Metadata): Option[Metadata] = { require(metadataTable.value.isTable, s"Not a table: $metadataTable") for { - width ← TikaAttributes.optional(metadataTable, TikaAttributes.ImageWidth) + width ← TikaAttributes.optional(metadataTable, TikaAttributes.ImageWidth) height ← TikaAttributes.optional(metadataTable, TikaAttributes.ImageHeight) - } yield Metadata(createTag(MDDisposition.METADATA), - Metadata.Value.ImageData(Metadata.ImageData(width.toInt, height.toInt))) + } yield Metadata(createTag(MDDisposition.METADATA), Metadata.Value.ImageData(Metadata.ImageData(width.toInt, height.toInt))) } def toDescription(metadataTable: Metadata): Option[Metadata] = { require(metadataTable.value.isTable, s"Not a table: $metadataTable") - val title = TikaAttributes.optional(metadataTable, TikaAttributes.Title) + val title = TikaAttributes.optional(metadataTable, TikaAttributes.Title) val description = TikaAttributes.optional(metadataTable, TikaAttributes.Description) Some((title.toSeq ++ description).mkString("\r\n\r\n").trim) diff --git a/metadata/tika/src/main/scala/com/karasiq/shadowcloud/metadata/tika/utils/TikaUtils.scala b/metadata/tika/src/main/scala/com/karasiq/shadowcloud/metadata/tika/utils/TikaUtils.scala index ddbfdfe1..bcb1c457 100644 --- a/metadata/tika/src/main/scala/com/karasiq/shadowcloud/metadata/tika/utils/TikaUtils.scala +++ b/metadata/tika/src/main/scala/com/karasiq/shadowcloud/metadata/tika/utils/TikaUtils.scala @@ -10,7 +10,7 @@ private[tika] object TikaUtils { val PluginId = "tika" def getHtmlContent(xhtml: String): String = { - val document = Jsoup.parse(xhtml) + val document = Jsoup.parse(xhtml) val documentBody = document.body() documentBody.html() } @@ -18,7 +18,10 @@ private[tika] object TikaUtils { def getHtmlMetaValue(xhtml: String, name: String): Option[String] = { import scala.collection.JavaConverters._ val document = Jsoup.parse(xhtml) - val element = document.head().children().asScala + val element = document + .head() + .children() + .asScala .find(e ⇒ e.tagName() == "meta" && e.attr("name") == name) element.map(_.attr("content")).filter(_.nonEmpty) } @@ -44,25 +47,22 @@ private[tika] object TikaUtils { def parseTimeString(timeString: String): Long = { import java.time.ZonedDateTime - ZonedDateTime.parse(timeString) - .toInstant - .toEpochMilli + ZonedDateTime.parse(timeString).toInstant.toEpochMilli } object ContentWrapper { def apply(parserId: String, textLimit: Int, xhtmlEnabled: Boolean) = { new TikaUtils.ContentWrapper( - TikaUtils.PluginId, parserId, + TikaUtils.PluginId, + parserId, Some(new BodyContentHandler(textLimit)).filter(_ ⇒ textLimit > 0), Some(new ToXMLContentHandler()).filter(_ ⇒ xhtmlEnabled) ) } } - final class ContentWrapper(plugin: String, parser: String, - textHandler: Option[ContentHandler], - xhtmlHandler: Option[ContentHandler]) { - private[this] final class TeeContentHandlerWrapper(subHandlers: Seq[ContentHandler]) extends TeeContentHandler(subHandlers:_*) { + final class ContentWrapper(plugin: String, parser: String, textHandler: Option[ContentHandler], xhtmlHandler: Option[ContentHandler]) { + private[this] final class TeeContentHandlerWrapper(subHandlers: Seq[ContentHandler]) extends TeeContentHandler(subHandlers: _*) { override def toString: String = "" // RecursiveParserWrapper fix } @@ -76,8 +76,9 @@ private[tika] object TikaUtils { handler .map(_.toString) .filter(TikaUtils.hasContent(_, format)) - .map(result ⇒ Metadata(Some(Metadata.Tag(plugin, parser, Metadata.Tag.Disposition.CONTENT)), - Metadata.Value.Text(Metadata.Text(format, result)))) + .map(result ⇒ + Metadata(Some(Metadata.Tag(plugin, parser, Metadata.Tag.Disposition.CONTENT)), Metadata.Value.Text(Metadata.Text(format, result))) + ) } Seq( @@ -86,4 +87,4 @@ private[tika] object TikaUtils { ).flatten } } -} \ No newline at end of file +} diff --git a/metadata/tika/src/test/scala/com/karasiq/shadowcloud/metadata/tika/TikaMetadataProviderTest.scala b/metadata/tika/src/test/scala/com/karasiq/shadowcloud/metadata/tika/TikaMetadataProviderTest.scala index 11a1ac5b..9a650313 100644 --- a/metadata/tika/src/test/scala/com/karasiq/shadowcloud/metadata/tika/TikaMetadataProviderTest.scala +++ b/metadata/tika/src/test/scala/com/karasiq/shadowcloud/metadata/tika/TikaMetadataProviderTest.scala @@ -14,11 +14,11 @@ import org.scalatest.{FlatSpecLike, SequentialNestedSuiteExecution} import scala.concurrent.duration._ class TikaMetadataProviderTest extends ActorSpec with ActorSpecImplicits with FlatSpecLike with SequentialNestedSuiteExecution { - val testPdfName = "TypeClasses.pdf" - val testPdfFile = new File(getClass.getClassLoader.getResource(testPdfName).toURI) + val testPdfName = "TypeClasses.pdf" + val testPdfFile = new File(getClass.getClassLoader.getResource(testPdfName).toURI) val testPdfBytes = ByteString.fromArrayUnsafe(FileUtils.readFileToByteArray(testPdfFile)) - val tika = new Tika() + val tika = new Tika() val detector = TikaMimeDetector(tika) "Mime detector" should "detect PDF" in { @@ -26,10 +26,11 @@ class TikaMetadataProviderTest extends ActorSpec with ActorSpecImplicits with Fl } val autoParserConfig = system.settings.config.getConfig("shadowcloud.metadata.tika.auto-parser") - val autoParser = TikaAutoParser(tika, autoParserConfig) + val autoParser = TikaAutoParser(tika, autoParserConfig) "Parser" should "extract text" in { - val stream = FileIO.fromPath(testPdfFile.toPath) + val stream = FileIO + .fromPath(testPdfFile.toPath) .via(autoParser.parseMetadata(testPdfName, "application/pdf")) .toMat(TestSink.probe)(Keep.right) .run() diff --git a/model/.jvm/src/main/scala/com/karasiq/shadowcloud/config/ConfigProps.scala b/model/.jvm/src/main/scala/com/karasiq/shadowcloud/config/ConfigProps.scala index c5302dd6..c379ea1e 100644 --- a/model/.jvm/src/main/scala/com/karasiq/shadowcloud/config/ConfigProps.scala +++ b/model/.jvm/src/main/scala/com/karasiq/shadowcloud/config/ConfigProps.scala @@ -8,10 +8,11 @@ import scala.collection.JavaConverters._ private[shadowcloud] object ConfigProps { private[this] val SupportedFormats = Set(SerializedProps.DefaultFormat, SerializedProps.JsonFormat) - + private[this] val JsonOptions = ConfigRenderOptions.concise() private[this] val HoconOptions = { - ConfigRenderOptions.defaults() + ConfigRenderOptions + .defaults() .setOriginComments(false) .setJson(false) } diff --git a/model/.jvm/src/main/scala/com/karasiq/shadowcloud/config/CryptoProps.scala b/model/.jvm/src/main/scala/com/karasiq/shadowcloud/config/CryptoProps.scala index 8f379b78..d0a03af5 100644 --- a/model/.jvm/src/main/scala/com/karasiq/shadowcloud/config/CryptoProps.scala +++ b/model/.jvm/src/main/scala/com/karasiq/shadowcloud/config/CryptoProps.scala @@ -6,32 +6,32 @@ import com.karasiq.shadowcloud.model.crypto.{EncryptionMethod, HashingMethod, Si private[shadowcloud] object CryptoProps extends ConfigImplicits { def hashing(config: Config): HashingMethod = { val algorithm = config.getString("algorithm") - val provider = config.withDefault("", _.getString("provider")) - val props = ConfigProps.fromConfig(withoutPaths(config, "algorithm", "provider")) + val provider = config.withDefault("", _.getString("provider")) + val props = ConfigProps.fromConfig(withoutPaths(config, "algorithm", "provider")) HashingMethod(algorithm, props, provider) } def encryption(config: Config): EncryptionMethod = { val algorithm = config.getString("algorithm") - val keySize = config.withDefault(256, _.getInt("key-size")) - val provider = config.withDefault("", _.getString("provider")) - val props = ConfigProps.fromConfig(withoutPaths(config, "algorithm", "key-size", "provider")) + val keySize = config.withDefault(256, _.getInt("key-size")) + val provider = config.withDefault("", _.getString("provider")) + val props = ConfigProps.fromConfig(withoutPaths(config, "algorithm", "key-size", "provider")) EncryptionMethod(algorithm, keySize, props, provider) } def signing(config: Config): SignMethod = { val algorithm = config.getString("algorithm") - val hashing = CryptoProps.hashing(config.getConfigOrRef("hashing")) - val keySize = config.withDefault(256, _.getInt("key-size")) - val provider = config.withDefault("", _.getString("provider")) - val props = ConfigProps.fromConfig(withoutPaths(config, "algorithm", "hashing", "key-size", "provider")) + val hashing = CryptoProps.hashing(config.getConfigOrRef("hashing")) + val keySize = config.withDefault(256, _.getInt("key-size")) + val provider = config.withDefault("", _.getString("provider")) + val props = ConfigProps.fromConfig(withoutPaths(config, "algorithm", "hashing", "key-size", "provider")) SignMethod(algorithm, hashing, keySize, props, provider) } def keyGeneration(props: SerializedProps): (Option[EncryptionMethod], Option[SignMethod]) = { val encConfig = ConfigProps.toConfig(props) - val encMethod = encConfig.optional(_.getConfig("encryption")).map(this.encryption) + val encMethod = encConfig.optional(_.getConfig("encryption")).map(this.encryption) val signMethod = encConfig.optional(_.getConfig("signing")).map(this.signing) (encMethod, signMethod) diff --git a/model/.jvm/src/main/scala/com/karasiq/shadowcloud/providers/LifecycleHook.scala b/model/.jvm/src/main/scala/com/karasiq/shadowcloud/providers/LifecycleHook.scala index 88292de9..b2d91091 100644 --- a/model/.jvm/src/main/scala/com/karasiq/shadowcloud/providers/LifecycleHook.scala +++ b/model/.jvm/src/main/scala/com/karasiq/shadowcloud/providers/LifecycleHook.scala @@ -6,13 +6,13 @@ trait LifecycleHook { } object LifecycleHook { - def initialize(f: => Unit): LifecycleHook = new LifecycleHook { + def initialize(f: ⇒ Unit): LifecycleHook = new LifecycleHook { override def initialize(): Unit = f - override def shutdown(): Unit = () + override def shutdown(): Unit = () } - def shutdown(f: => Unit): LifecycleHook = new LifecycleHook { + def shutdown(f: ⇒ Unit): LifecycleHook = new LifecycleHook { override def initialize(): Unit = () - override def shutdown(): Unit = f + override def shutdown(): Unit = f } } diff --git a/model/src/main/scala/com/karasiq/shadowcloud/config/SerializedProps.scala b/model/src/main/scala/com/karasiq/shadowcloud/config/SerializedProps.scala index 1f22e105..acbeb2ed 100644 --- a/model/src/main/scala/com/karasiq/shadowcloud/config/SerializedProps.scala +++ b/model/src/main/scala/com/karasiq/shadowcloud/config/SerializedProps.scala @@ -31,7 +31,7 @@ final case class SerializedProps(format: String = "", data: ByteString = ByteStr object SerializedProps { val empty = SerializedProps() - val EmptyFormat = "" + val EmptyFormat = "" val DefaultFormat = "hocon" - val JsonFormat = "json" -} \ No newline at end of file + val JsonFormat = "json" +} diff --git a/model/src/main/scala/com/karasiq/shadowcloud/config/WrappedConfig.scala b/model/src/main/scala/com/karasiq/shadowcloud/config/WrappedConfig.scala index 8ac19c5b..d3343563 100644 --- a/model/src/main/scala/com/karasiq/shadowcloud/config/WrappedConfig.scala +++ b/model/src/main/scala/com/karasiq/shadowcloud/config/WrappedConfig.scala @@ -10,4 +10,4 @@ trait WrappedConfig extends SCEntity { trait WrappedConfigFactory[T <: WrappedConfig] { def apply(config: Config): T -} \ No newline at end of file +} diff --git a/model/src/main/scala/com/karasiq/shadowcloud/exceptions/CryptoException.scala b/model/src/main/scala/com/karasiq/shadowcloud/exceptions/CryptoException.scala index 21a93261..df5dec6f 100644 --- a/model/src/main/scala/com/karasiq/shadowcloud/exceptions/CryptoException.scala +++ b/model/src/main/scala/com/karasiq/shadowcloud/exceptions/CryptoException.scala @@ -7,11 +7,11 @@ import com.karasiq.common.encoding.HexString sealed abstract class CryptoException(message: String = null, cause: Throwable = null) extends SCException(message, cause) object CryptoException { - final case class ChecksumError(expected: ByteString, actual: ByteString) extends CryptoException( - "Checksum not match: " + HexString.encode(expected) + " / " + HexString.encode(actual)) - final case class EncryptError(cause: Throwable = null) extends CryptoException("Encryption error", cause) - final case class DecryptError(cause: Throwable = null) extends CryptoException("Decryption error", cause) + final case class ChecksumError(expected: ByteString, actual: ByteString) + extends CryptoException("Checksum not match: " + HexString.encode(expected) + " / " + HexString.encode(actual)) + final case class EncryptError(cause: Throwable = null) extends CryptoException("Encryption error", cause) + final case class DecryptError(cause: Throwable = null) extends CryptoException("Decryption error", cause) final case class InvalidSignature(cause: Throwable = null) extends CryptoException("Invalid signature", cause) - final case class KeyMissing(cause: Throwable = null) extends CryptoException("No appropriate key found", cause) - final case class ReuseError(cause: Throwable = null) extends CryptoException("Crypto parameters illegal reuse") -} \ No newline at end of file + final case class KeyMissing(cause: Throwable = null) extends CryptoException("No appropriate key found", cause) + final case class ReuseError(cause: Throwable = null) extends CryptoException("Crypto parameters illegal reuse") +} diff --git a/model/src/main/scala/com/karasiq/shadowcloud/exceptions/RegionException.scala b/model/src/main/scala/com/karasiq/shadowcloud/exceptions/RegionException.scala index eede992a..43362a5a 100644 --- a/model/src/main/scala/com/karasiq/shadowcloud/exceptions/RegionException.scala +++ b/model/src/main/scala/com/karasiq/shadowcloud/exceptions/RegionException.scala @@ -3,43 +3,44 @@ package com.karasiq.shadowcloud.exceptions import com.karasiq.shadowcloud.index.diffs.IndexDiff import com.karasiq.shadowcloud.model.{Chunk, Path} -sealed abstract class RegionException(message: String = null, cause: Throwable = null) - extends SCException(message, cause) +sealed abstract class RegionException(message: String = null, cause: Throwable = null) extends SCException(message, cause) object RegionException { // ----------------------------------------------------------------------- // Not found errors // ----------------------------------------------------------------------- - final case class ChunkNotFound(chunk: Chunk) - extends RegionException("Chunk not found: " + chunk) with SCException.NotFound + final case class ChunkNotFound(chunk: Chunk) extends RegionException("Chunk not found: " + chunk) with SCException.NotFound - final case class FileNotFound(path: Path) - extends RegionException("File not found: " + path) with SCException.NotFound + final case class FileNotFound(path: Path) extends RegionException("File not found: " + path) with SCException.NotFound - final case class DirectoryNotFound(path: Path) - extends RegionException("Directory not found: " + path) with SCException.NotFound + final case class DirectoryNotFound(path: Path) extends RegionException("Directory not found: " + path) with SCException.NotFound // ----------------------------------------------------------------------- // IO errors // ----------------------------------------------------------------------- sealed abstract class ChunkIOError(message: String = null, cause: Throwable = null) - extends RegionException(message, cause) with SCException.IOError with SCException.ChunkAssociated + extends RegionException(message, cause) + with SCException.IOError + with SCException.ChunkAssociated - sealed abstract class IndexIOError(message: String = null, cause: Throwable = null) - extends RegionException(message, cause) with SCException.IOError + sealed abstract class IndexIOError(message: String = null, cause: Throwable = null) extends RegionException(message, cause) with SCException.IOError final case class ChunkWriteFailed(chunk: Chunk, cause: Throwable) - extends ChunkIOError("Chunk write failed: " + chunk, cause) with SCException.WrappedError + extends ChunkIOError("Chunk write failed: " + chunk, cause) + with SCException.WrappedError final case class ChunkReadFailed(chunk: Chunk, cause: Throwable) - extends ChunkIOError("Chunk read failed: " + chunk, cause) with SCException.WrappedError + extends ChunkIOError("Chunk read failed: " + chunk, cause) + with SCException.WrappedError final case class ChunkRepairFailed(chunk: Chunk, cause: Throwable) - extends ChunkIOError("Chunk repair failed: " + chunk, cause) with SCException.WrappedError + extends ChunkIOError("Chunk repair failed: " + chunk, cause) + with SCException.WrappedError final case class IndexWriteFailed(diff: IndexDiff, cause: Throwable) - extends IndexIOError("Index write failed: " + diff, cause) with SCException.WrappedError with SCException.DiffAssociated + extends IndexIOError("Index write failed: " + diff, cause) + with SCException.WrappedError + with SCException.DiffAssociated - final case class ChunkUnavailable(chunk: Chunk) - extends ChunkIOError("Chunk unavailable: " + chunk) -} \ No newline at end of file + final case class ChunkUnavailable(chunk: Chunk) extends ChunkIOError("Chunk unavailable: " + chunk) +} diff --git a/model/src/main/scala/com/karasiq/shadowcloud/exceptions/SCExceptions.scala b/model/src/main/scala/com/karasiq/shadowcloud/exceptions/SCExceptions.scala index 898b783d..7e87e545 100644 --- a/model/src/main/scala/com/karasiq/shadowcloud/exceptions/SCExceptions.scala +++ b/model/src/main/scala/com/karasiq/shadowcloud/exceptions/SCExceptions.scala @@ -4,15 +4,17 @@ import com.karasiq.shadowcloud.index.diffs.IndexDiff import com.karasiq.shadowcloud.model.Chunk object SCExceptions { - final case class ChunkDataIsEmpty(chunk: Chunk) - extends SCException("Chunk data is empty: " + chunk) with SCException.ChunkAssociated + final case class ChunkDataIsEmpty(chunk: Chunk) extends SCException("Chunk data is empty: " + chunk) with SCException.ChunkAssociated final case class ChunkConflict(chunk: Chunk, offeredChunk: Chunk) - extends SCException("Chunk conflict: " + chunk + " / " + offeredChunk) with SCException.ChunkAssociated + extends SCException("Chunk conflict: " + chunk + " / " + offeredChunk) + with SCException.ChunkAssociated final case class ChunkVerifyError(chunk: Chunk, cause: Throwable = null) - extends SCException("Chunk verify error: " + chunk, cause) with SCException.ChunkAssociated + extends SCException("Chunk verify error: " + chunk, cause) + with SCException.ChunkAssociated final case class DiffConflict(diff: IndexDiff, offeredDiff: IndexDiff) - extends SCException("Diff conflict: " + diff + " / " + offeredDiff) with SCException.DiffAssociated + extends SCException("Diff conflict: " + diff + " / " + offeredDiff) + with SCException.DiffAssociated } diff --git a/model/src/main/scala/com/karasiq/shadowcloud/exceptions/StorageException.scala b/model/src/main/scala/com/karasiq/shadowcloud/exceptions/StorageException.scala index 687950a9..bfa1a4dc 100644 --- a/model/src/main/scala/com/karasiq/shadowcloud/exceptions/StorageException.scala +++ b/model/src/main/scala/com/karasiq/shadowcloud/exceptions/StorageException.scala @@ -3,15 +3,16 @@ package com.karasiq.shadowcloud.exceptions import com.karasiq.shadowcloud.model.Path sealed abstract class StorageException(message: String = null, cause: Throwable = null) - extends SCException(message, cause) with SCException.IOError with SCException.PathAssociated + extends SCException(message, cause) + with SCException.IOError + with SCException.PathAssociated object StorageException { - final case class NotFound(path: Path, cause: Throwable = null) - extends StorageException("Not found: " + path, cause) with SCException.NotFound + final case class NotFound(path: Path, cause: Throwable = null) extends StorageException("Not found: " + path, cause) with SCException.NotFound final case class AlreadyExists(path: Path, cause: Throwable = null) - extends StorageException("Already exists: " + path, cause) with SCException.AlreadyExists + extends StorageException("Already exists: " + path, cause) + with SCException.AlreadyExists - final case class IOFailure(path: Path, cause: Throwable) - extends StorageException("IO failure: " + path, cause) with SCException.WrappedError + final case class IOFailure(path: Path, cause: Throwable) extends StorageException("IO failure: " + path, cause) with SCException.WrappedError } diff --git a/model/src/main/scala/com/karasiq/shadowcloud/exceptions/SupervisorException.scala b/model/src/main/scala/com/karasiq/shadowcloud/exceptions/SupervisorException.scala index 47c64f0a..98ce98fd 100644 --- a/model/src/main/scala/com/karasiq/shadowcloud/exceptions/SupervisorException.scala +++ b/model/src/main/scala/com/karasiq/shadowcloud/exceptions/SupervisorException.scala @@ -2,19 +2,26 @@ package com.karasiq.shadowcloud.exceptions import com.karasiq.shadowcloud.model.{RegionId, StorageId} -sealed abstract class SupervisorException(message: String = null, cause: Throwable = null) - extends SCException(message, cause) +sealed abstract class SupervisorException(message: String = null, cause: Throwable = null) extends SCException(message, cause) object SupervisorException { final case class RegionNotFound(regionId: RegionId) - extends SupervisorException("Region not found: " + regionId) with SCException.NotFound with SCException.RegionAssociated + extends SupervisorException("Region not found: " + regionId) + with SCException.NotFound + with SCException.RegionAssociated final case class StorageNotFound(storageId: StorageId) - extends SupervisorException("Storage not found: " + storageId) with SCException.NotFound with SCException.StorageAssociated + extends SupervisorException("Storage not found: " + storageId) + with SCException.NotFound + with SCException.StorageAssociated final case class IllegalRegionState(regionId: RegionId, cause: Throwable = null) - extends SupervisorException("Illegal region state: " + regionId) with SCException.WrappedError with SCException.RegionAssociated + extends SupervisorException("Illegal region state: " + regionId) + with SCException.WrappedError + with SCException.RegionAssociated final case class IllegalStorageState(storageId: StorageId, cause: Throwable = null) - extends SupervisorException("Illegal storage state: " + storageId) with SCException.WrappedError with SCException.StorageAssociated + extends SupervisorException("Illegal storage state: " + storageId) + with SCException.WrappedError + with SCException.StorageAssociated } diff --git a/model/src/main/scala/com/karasiq/shadowcloud/index/ChunkIndex.scala b/model/src/main/scala/com/karasiq/shadowcloud/index/ChunkIndex.scala index a5b7bd28..4799ba7e 100644 --- a/model/src/main/scala/com/karasiq/shadowcloud/index/ChunkIndex.scala +++ b/model/src/main/scala/com/karasiq/shadowcloud/index/ChunkIndex.scala @@ -8,10 +8,9 @@ import com.karasiq.shadowcloud.utils.Utils import scala.collection.GenTraversableOnce @SerialVersionUID(0L) -final case class ChunkIndex(chunks: Set[Chunk] = Set.empty) - extends SCEntity with Mergeable with HasEmpty with HasWithoutData { +final case class ChunkIndex(chunks: Set[Chunk] = Set.empty) extends SCEntity with Mergeable with HasEmpty with HasWithoutData { - type Repr = ChunkIndex + type Repr = ChunkIndex type DiffRepr = ChunkIndexDiff def contains(chunk: Chunk): Boolean = { diff --git a/model/src/main/scala/com/karasiq/shadowcloud/index/FolderIndex.scala b/model/src/main/scala/com/karasiq/shadowcloud/index/FolderIndex.scala index fcc8fc54..61538120 100644 --- a/model/src/main/scala/com/karasiq/shadowcloud/index/FolderIndex.scala +++ b/model/src/main/scala/com/karasiq/shadowcloud/index/FolderIndex.scala @@ -10,9 +10,12 @@ import scala.collection.{GenTraversableOnce, mutable} @SerialVersionUID(0L) final case class FolderIndex(folders: Map[Path, Folder] = Map(Path.root → Folder(Path.root))) - extends SCEntity with Mergeable with HasEmpty with HasWithoutData { + extends SCEntity + with Mergeable + with HasEmpty + with HasWithoutData { - type Repr = FolderIndex + type Repr = FolderIndex type DiffRepr = FolderIndexDiff require(folders.contains(Path.root), "No root directory") @@ -48,8 +51,8 @@ final case class FolderIndex(folders: Map[Path, Folder] = Map(Path.root → Fold def addFolders(folders: GenTraversableOnce[Folder]): FolderIndex = { val diffs = folders.toIterator.flatMap { folder ⇒ val existing = this.folders.get(folder.path) - val diff = if (existing.isEmpty) FolderDiff.create(folder) else folder.diff(existing.get) - val parent = this.folders.get(folder.path.parent) + val diff = if (existing.isEmpty) FolderDiff.create(folder) else folder.diff(existing.get) + val parent = this.folders.get(folder.path.parent) if (folder.path.isRoot || parent.exists(_.folders.contains(folder.path.name))) { Iterator.single(diff) } else { @@ -115,12 +118,13 @@ final case class FolderIndex(folders: Map[Path, Folder] = Map(Path.root → Fold private[this] def applyDiffs(diffs: GenTraversableOnce[FolderDiff]): FolderIndex = { if (diffs.isEmpty) return this - + val modified = mutable.AnyRefMap[Path, Folder]() - val deleted = mutable.Set[Path]() + val deleted = mutable.Set[Path]() def getFolder(path: Path) = { - modified.get(path) + modified + .get(path) .orElse(this.get(path)) } @@ -164,7 +168,6 @@ final case class FolderIndex(folders: Map[Path, Folder] = Map(Path.root → Fold modified += diff.path → folder.patch(diff) } - diff.newFolders .map(diff.path / _) .filterNot(path ⇒ folders.contains(path) || modified.contains(path)) @@ -197,7 +200,9 @@ object FolderIndex { Iterator.single(folder) ++ subFolders.flatMap(traverseFolderTreeRec(index, _)) } - index.get(path).iterator + index + .get(path) + .iterator .flatMap(traverseFolderTreeRec(index, _)) } } diff --git a/model/src/main/scala/com/karasiq/shadowcloud/index/diffs/ChunkIndexDiff.scala b/model/src/main/scala/com/karasiq/shadowcloud/index/diffs/ChunkIndexDiff.scala index f582a1d3..90f6b444 100644 --- a/model/src/main/scala/com/karasiq/shadowcloud/index/diffs/ChunkIndexDiff.scala +++ b/model/src/main/scala/com/karasiq/shadowcloud/index/diffs/ChunkIndexDiff.scala @@ -1,7 +1,5 @@ package com.karasiq.shadowcloud.index.diffs - - import com.karasiq.shadowcloud.index.ChunkIndex import com.karasiq.shadowcloud.index.utils.{HasEmpty, HasWithoutData, MergeableDiff} import com.karasiq.shadowcloud.model.{Chunk, SCEntity} @@ -10,20 +8,21 @@ import com.karasiq.shadowcloud.utils.{MergeUtil, Utils} @SerialVersionUID(0L) final case class ChunkIndexDiff(newChunks: Set[Chunk] = Set.empty, deletedChunks: Set[Chunk] = Set.empty) - extends SCEntity with MergeableDiff with HasEmpty with HasWithoutData { - + extends SCEntity + with MergeableDiff + with HasEmpty + with HasWithoutData { + type Repr = ChunkIndexDiff // Delete wins by default def mergeWith(diff: ChunkIndexDiff, decider: SplitDecider[Chunk] = SplitDecider.dropDuplicates): ChunkIndexDiff = { - val (newChunks, deletedChunks) = MergeUtil.splitSets(this.newChunks ++ diff.newChunks, - this.deletedChunks ++ diff.deletedChunks, decider) + val (newChunks, deletedChunks) = MergeUtil.splitSets(this.newChunks ++ diff.newChunks, this.deletedChunks ++ diff.deletedChunks, decider) withChunks(newChunks, deletedChunks) } def diffWith(diff: ChunkIndexDiff, decider: Decider[Chunk] = Decider.diff): ChunkIndexDiff = { - withChunks(MergeUtil.mergeSets(this.newChunks, diff.newChunks, decider), - MergeUtil.mergeSets(this.deletedChunks, diff.deletedChunks, decider)) + withChunks(MergeUtil.mergeSets(this.newChunks, diff.newChunks, decider), MergeUtil.mergeSets(this.deletedChunks, diff.deletedChunks, decider)) } def merge(right: ChunkIndexDiff): ChunkIndexDiff = { @@ -41,7 +40,7 @@ final case class ChunkIndexDiff(newChunks: Set[Chunk] = Set.empty, deletedChunks def deletes: ChunkIndexDiff = { withChunks(newChunks = Set.empty) } - + def creates: ChunkIndexDiff = { withChunks(deletedChunks = Set.empty) } diff --git a/model/src/main/scala/com/karasiq/shadowcloud/index/diffs/FolderDiff.scala b/model/src/main/scala/com/karasiq/shadowcloud/index/diffs/FolderDiff.scala index 9c20d854..18e71059 100644 --- a/model/src/main/scala/com/karasiq/shadowcloud/index/diffs/FolderDiff.scala +++ b/model/src/main/scala/com/karasiq/shadowcloud/index/diffs/FolderDiff.scala @@ -1,22 +1,31 @@ package com.karasiq.shadowcloud.index.diffs - - import com.karasiq.shadowcloud.index.utils._ import com.karasiq.shadowcloud.model.{File, Folder, Path, SCEntity} import com.karasiq.shadowcloud.utils.MergeUtil.SplitDecider import com.karasiq.shadowcloud.utils.{MergeUtil, Utils} @SerialVersionUID(0L) -final case class FolderDiff(path: Path, time: Long = 0, newFiles: Set[File] = Set.empty, - deletedFiles: Set[File] = Set.empty, newFolders: Set[String] = Set.empty, - deletedFolders: Set[String] = Set.empty) - extends SCEntity with HasPath with MergeableDiff with HasEmpty with HasWithoutData { +final case class FolderDiff( + path: Path, + time: Long = 0, + newFiles: Set[File] = Set.empty, + deletedFiles: Set[File] = Set.empty, + newFolders: Set[String] = Set.empty, + deletedFolders: Set[String] = Set.empty +) extends SCEntity + with HasPath + with MergeableDiff + with HasEmpty + with HasWithoutData { type Repr = FolderDiff - def mergeWith(diff: FolderDiff, diffDecider: FolderDiffDecider = FolderDiffDecider.rightWins, - folderDecider: FolderDecider = FolderDecider.mutualExclude): FolderDiff = { + def mergeWith( + diff: FolderDiff, + diffDecider: FolderDiffDecider = FolderDiffDecider.rightWins, + folderDecider: FolderDecider = FolderDecider.mutualExclude + ): FolderDiff = { def unifyPaths(files: Set[File]): Set[File] = files.map(f ⇒ f.copy(path = f.path.withParent(this.path))) require(diff.path == path, "Invalid path") @@ -31,9 +40,11 @@ final case class FolderDiff(path: Path, time: Long = 0, newFiles: Set[File] = Se copy(path, newTimestamp, unifyPaths(newFiles), unifyPaths(deletedFiles), newFolders, deletedFolders) } - def diffWith(oldDiff: FolderDiff, - diffDecider: FolderDiffDecider = FolderDiffDecider.idempotent, - folderDecider: FolderDecider = FolderDecider.mutualExclude): FolderDiff = { + def diffWith( + oldDiff: FolderDiff, + diffDecider: FolderDiffDecider = FolderDiffDecider.idempotent, + folderDecider: FolderDecider = FolderDecider.mutualExclude + ): FolderDiff = { require(oldDiff.path == path, "Invalid path") mergeWith(oldDiff.reverse, diffDecider, folderDecider) } @@ -68,7 +79,8 @@ final case class FolderDiff(path: Path, time: Long = 0, newFiles: Set[File] = Se override def toString: String = { if (nonEmpty) { - s"FolderDiff($path, $time, new files = [${Utils.printValues(newFiles, 5)}], deleted files = [${Utils.printValues(deletedFiles, 10)}], new folders = [${Utils.printValues(newFolders, 10)}], deleted folders = [${Utils.printValues(deletedFolders, 10)}])" + s"FolderDiff($path, $time, new files = [${Utils.printValues(newFiles, 5)}], deleted files = [${Utils.printValues(deletedFiles, 10)}], new folders = [${Utils + .printValues(newFolders, 10)}], deleted folders = [${Utils.printValues(deletedFolders, 10)}])" } else { s"FolderDiff.empty($path)" } @@ -78,7 +90,7 @@ final case class FolderDiff(path: Path, time: Long = 0, newFiles: Set[File] = Se object FolderDiff { def apply(oldFolder: Folder, newFolder: Folder): FolderDiff = { require(oldFolder.path == newFolder.path, "Invalid path") - val (leftFiles, rightFiles) = MergeUtil.splitSets(oldFolder.files, newFolder.files, SplitDecider.dropDuplicates) + val (leftFiles, rightFiles) = MergeUtil.splitSets(oldFolder.files, newFolder.files, SplitDecider.dropDuplicates) val (leftFolders, rightFolders) = MergeUtil.splitSets(oldFolder.folders, newFolder.folders, SplitDecider.dropDuplicates) FolderDiff(oldFolder.path, newFolder.timestamp.lastModified, rightFiles, leftFiles, rightFolders, leftFolders) } diff --git a/model/src/main/scala/com/karasiq/shadowcloud/index/diffs/FolderIndexDiff.scala b/model/src/main/scala/com/karasiq/shadowcloud/index/diffs/FolderIndexDiff.scala index de82b89b..b37843ef 100644 --- a/model/src/main/scala/com/karasiq/shadowcloud/index/diffs/FolderIndexDiff.scala +++ b/model/src/main/scala/com/karasiq/shadowcloud/index/diffs/FolderIndexDiff.scala @@ -1,7 +1,5 @@ package com.karasiq.shadowcloud.index.diffs - - import com.karasiq.shadowcloud.index.FolderIndex import com.karasiq.shadowcloud.index.utils._ import com.karasiq.shadowcloud.model.{File, Folder, Path, SCEntity} @@ -10,28 +8,35 @@ import com.karasiq.shadowcloud.utils.MergeUtil.State.{Conflict, Equal, Left, Rig import com.karasiq.shadowcloud.utils.{MergeUtil, Utils} @SerialVersionUID(0L) -final case class FolderIndexDiff(folders: Seq[FolderDiff] = Vector.empty) - extends SCEntity with MergeableDiff with HasEmpty with HasWithoutData { +final case class FolderIndexDiff(folders: Seq[FolderDiff] = Vector.empty) extends SCEntity with MergeableDiff with HasEmpty with HasWithoutData { type Repr = FolderIndexDiff // Delete wins by default - def mergeWith(diff: FolderIndexDiff, diffDecider: FolderDiffDecider = FolderDiffDecider.rightWins, - folderDecider: FolderDecider = FolderDecider.mutualExclude): FolderIndexDiff = { - val folders = MergeUtil.mergeByKey[Path, FolderDiff](this.folders, diff.folders, _.path, { - case Conflict(left, right) ⇒ + def mergeWith( + diff: FolderIndexDiff, + diffDecider: FolderDiffDecider = FolderDiffDecider.rightWins, + folderDecider: FolderDecider = FolderDecider.mutualExclude + ): FolderIndexDiff = { + val folders = MergeUtil.mergeByKey[Path, FolderDiff]( + this.folders, + diff.folders, + _.path, + { case Conflict(left, right) ⇒ Some(left.mergeWith(right, diffDecider, folderDecider)) - }) + } + ) withFolders(folders) } - def diffWith(diff: FolderIndexDiff, - decider: Decider[FolderDiff] = Decider.diff, - diffDecider: FolderDiffDecider = FolderDiffDecider.idempotent, - folderDecider: FolderDecider = FolderDecider.mutualExclude): FolderIndexDiff = { - val decider1: Decider[FolderDiff] = decider.orElse { - case Conflict(left, right) ⇒ - Some(right.diffWith(left, diffDecider, folderDecider)) + def diffWith( + diff: FolderIndexDiff, + decider: Decider[FolderDiff] = Decider.diff, + diffDecider: FolderDiffDecider = FolderDiffDecider.idempotent, + folderDecider: FolderDecider = FolderDecider.mutualExclude + ): FolderIndexDiff = { + val decider1: Decider[FolderDiff] = decider.orElse { case Conflict(left, right) ⇒ + Some(right.diffWith(left, diffDecider, folderDecider)) } val folders = MergeUtil.mergeByKey[Path, FolderDiff](this.folders, diff.folders, _.path, decider1) withFolders(folders) @@ -177,15 +182,17 @@ object FolderIndexDiff { // Explicitly deletes folder items def deleteFolderItems(index: FolderIndex, path: Path): FolderIndexDiff = { - val folders = index.get(path).toSeq + val folders = index + .get(path) + .toSeq .flatMap(f ⇒ f +: f.folders.toSeq.map(f.path / _).flatMap(index.get)) - deleteFolders(folders:_*) + deleteFolders(folders: _*) } // Explicitly deletes folder tree def deleteFolderTree(index: FolderIndex, path: Path): FolderIndexDiff = { val folders = FolderIndex.traverseFolderTree(index, path) - deleteFolders(folders.toSeq:_*) + deleteFolders(folders.toSeq: _*) } def deleteFiles(files: File*): FolderIndexDiff = { @@ -213,13 +220,13 @@ object FolderIndexDiff { def copyFile(index: FolderIndex, path: Path, newPath: Path): FolderIndexDiff = { require(!path.isRoot && !newPath.isRoot, "Invalid file path") val newFiles = index.getFiles(path).map(f ⇒ f.copy(path = newPath)) - val diff = FolderDiff(newPath.parent, Utils.timestamp, newFiles = newFiles) + val diff = FolderDiff(newPath.parent, Utils.timestamp, newFiles = newFiles) FolderIndexDiff.fromDiffs(diff) } def moveFiles(files: Set[File], newPath: Path): FolderIndexDiff = { - val deleteDiff = deleteFiles(files.toSeq:_*) - val copyDiff = copyFiles(files, newPath) + val deleteDiff = deleteFiles(files.toSeq: _*) + val copyDiff = copyFiles(files, newPath) deleteDiff.mergeWith(copyDiff, FolderDiffDecider.rightWins, FolderDecider.createWins) } @@ -229,7 +236,7 @@ object FolderIndexDiff { def moveFile(index: FolderIndex, path: Path, newPath: Path): FolderIndexDiff = { val deleteDiff = deleteFiles(index.getFiles(path).toSeq: _*) - val copyDiff = copyFile(index, path, newPath) + val copyDiff = copyFile(index, path, newPath) deleteDiff.mergeWith(copyDiff, FolderDiffDecider.rightWins, FolderDecider.createWins) } @@ -239,20 +246,20 @@ object FolderIndexDiff { } def copyFolder(index: FolderIndex, path: Path, newPath: Path): FolderIndexDiff = { - val folders = FolderIndex.traverseFolderTree(index, path) + val folders = FolderIndex.traverseFolderTree(index, path) val newFolders = folders.map(f ⇒ f.withPath(newPath / f.path.toRelative(path))) - createFolders(newFolders.toSeq:_*) + createFolders(newFolders.toSeq: _*) } def moveFolder(folder: Folder, newPath: Path): FolderIndexDiff = { require(!folder.path.isRoot, "Cannot move root") val deleteDiff = deleteFolderPaths(folder.path) - val copyDiff = copyFolder(folder, newPath) + val copyDiff = copyFolder(folder, newPath) deleteDiff.mergeWith(copyDiff, FolderDiffDecider.rightWins, FolderDecider.createWins) } def moveFolder(index: FolderIndex, path: Path, newPath: Path): FolderIndexDiff = { - val copyDiff = copyFolder(index, path, newPath) + val copyDiff = copyFolder(index, path, newPath) val deleteDiff = deleteFolderPaths(path) deleteDiff.mergeWith(copyDiff, FolderDiffDecider.rightWins, FolderDecider.createWins) } diff --git a/model/src/main/scala/com/karasiq/shadowcloud/index/diffs/IndexDiff.scala b/model/src/main/scala/com/karasiq/shadowcloud/index/diffs/IndexDiff.scala index 3ae5bbec..87dd8d41 100644 --- a/model/src/main/scala/com/karasiq/shadowcloud/index/diffs/IndexDiff.scala +++ b/model/src/main/scala/com/karasiq/shadowcloud/index/diffs/IndexDiff.scala @@ -1,35 +1,41 @@ package com.karasiq.shadowcloud.index.diffs - - import com.karasiq.shadowcloud.index.utils._ import com.karasiq.shadowcloud.model.{Chunk, Folder, Path, SCEntity} import com.karasiq.shadowcloud.utils.MergeUtil.{Decider, SplitDecider} import com.karasiq.shadowcloud.utils.Utils @SerialVersionUID(0L) -final case class IndexDiff(time: Long = Utils.timestamp, - folders: FolderIndexDiff = FolderIndexDiff.empty, - chunks: ChunkIndexDiff = ChunkIndexDiff.empty) - extends SCEntity with MergeableDiff with HasEmpty with HasWithoutData { +final case class IndexDiff( + time: Long = Utils.timestamp, + folders: FolderIndexDiff = FolderIndexDiff.empty, + chunks: ChunkIndexDiff = ChunkIndexDiff.empty +) extends SCEntity + with MergeableDiff + with HasEmpty + with HasWithoutData { type Repr = IndexDiff // Delete wins by default - def mergeWith(diff: IndexDiff, folderDiffDecider: FolderDiffDecider = FolderDiffDecider.rightWins, - folderDecider: FolderDecider = FolderDecider.mutualExclude, - chunkDecider: SplitDecider[Chunk] = SplitDecider.dropDuplicates): IndexDiff = { + def mergeWith( + diff: IndexDiff, + folderDiffDecider: FolderDiffDecider = FolderDiffDecider.rightWins, + folderDecider: FolderDecider = FolderDecider.mutualExclude, + chunkDecider: SplitDecider[Chunk] = SplitDecider.dropDuplicates + ): IndexDiff = { val maxTime = math.max(time, diff.time) - withDiffs(folders.mergeWith(diff.folders, folderDiffDecider, folderDecider), - chunks.mergeWith(diff.chunks, chunkDecider), maxTime) + withDiffs(folders.mergeWith(diff.folders, folderDiffDecider, folderDecider), chunks.mergeWith(diff.chunks, chunkDecider), maxTime) } - def diffWith(diff: IndexDiff, decider: Decider[FolderDiff] = Decider.diff, - folderDiffDecider: FolderDiffDecider = FolderDiffDecider.idempotent, - folderDecider: FolderDecider = FolderDecider.mutualExclude, - chunkDecider: Decider[Chunk] = Decider.diff): IndexDiff = { - withDiffs(folders.diffWith(diff.folders, decider, folderDiffDecider, folderDecider), - chunks.diffWith(diff.chunks, chunkDecider)) + def diffWith( + diff: IndexDiff, + decider: Decider[FolderDiff] = Decider.diff, + folderDiffDecider: FolderDiffDecider = FolderDiffDecider.idempotent, + folderDecider: FolderDecider = FolderDecider.mutualExclude, + chunkDecider: Decider[Chunk] = Decider.diff + ): IndexDiff = { + withDiffs(folders.diffWith(diff.folders, decider, folderDiffDecider, folderDecider), chunks.diffWith(diff.chunks, chunkDecider)) } def merge(right: IndexDiff): IndexDiff = { @@ -64,9 +70,7 @@ final case class IndexDiff(time: Long = Utils.timestamp, if (isEmpty) "IndexDiff.empty" else s"IndexDiff($time, $folders, $chunks)" } - private[this] def withDiffs(folders: FolderIndexDiff = this.folders, - chunks: ChunkIndexDiff = this.chunks, - time: Long = this.time): IndexDiff = { + private[this] def withDiffs(folders: FolderIndexDiff = this.folders, chunks: ChunkIndexDiff = this.chunks, time: Long = this.time): IndexDiff = { if (folders.isEmpty && chunks.isEmpty) IndexDiff.empty else copy(time, folders, chunks) } } diff --git a/model/src/main/scala/com/karasiq/shadowcloud/index/utils/DiffMergeDecider.scala b/model/src/main/scala/com/karasiq/shadowcloud/index/utils/DiffMergeDecider.scala index e12daeb8..ef6e2b7f 100644 --- a/model/src/main/scala/com/karasiq/shadowcloud/index/utils/DiffMergeDecider.scala +++ b/model/src/main/scala/com/karasiq/shadowcloud/index/utils/DiffMergeDecider.scala @@ -4,33 +4,25 @@ import com.karasiq.shadowcloud.utils.MergeUtil import com.karasiq.shadowcloud.utils.MergeUtil.SplitDecider trait DiffMergeDecider[T] { - def apply(leftCreates: Set[T], leftDeletes: Set[T], - rightCreates: Set[T], rightDeletes: Set[T], - splitDecider: SplitDecider[T]): (Set[T], Set[T]) + def apply(leftCreates: Set[T], leftDeletes: Set[T], rightCreates: Set[T], rightDeletes: Set[T], splitDecider: SplitDecider[T]): (Set[T], Set[T]) } //noinspection ConvertExpressionToSAM object DiffMergeDecider { def idempotent[T]: DiffMergeDecider[T] = new DiffMergeDecider[T] { - def apply(leftCreates: Set[T], leftDeletes: Set[T], - rightCreates: Set[T], rightDeletes: Set[T], - splitDecider: SplitDecider[T]) = { + def apply(leftCreates: Set[T], leftDeletes: Set[T], rightCreates: Set[T], rightDeletes: Set[T], splitDecider: SplitDecider[T]) = { MergeUtil.splitSets(leftCreates ++ rightCreates, leftDeletes ++ rightDeletes, splitDecider) } } def leftWins[T]: DiffMergeDecider[T] = new DiffMergeDecider[T] { - def apply(leftCreates: Set[T], leftDeletes: Set[T], - rightCreates: Set[T], rightDeletes: Set[T], - splitDecider: SplitDecider[T]) = { + def apply(leftCreates: Set[T], leftDeletes: Set[T], rightCreates: Set[T], rightDeletes: Set[T], splitDecider: SplitDecider[T]) = { MergeUtil.splitSets(rightCreates -- leftDeletes ++ leftCreates, rightDeletes -- leftCreates ++ leftDeletes, splitDecider) } } def rightWins[T]: DiffMergeDecider[T] = new DiffMergeDecider[T] { - def apply(leftCreates: Set[T], leftDeletes: Set[T], - rightCreates: Set[T], rightDeletes: Set[T], - splitDecider: SplitDecider[T]) = { + def apply(leftCreates: Set[T], leftDeletes: Set[T], rightCreates: Set[T], rightDeletes: Set[T], splitDecider: SplitDecider[T]) = { leftWins[T](rightCreates, rightDeletes, leftCreates, leftDeletes, splitDecider) } } diff --git a/model/src/main/scala/com/karasiq/shadowcloud/index/utils/FolderDecider.scala b/model/src/main/scala/com/karasiq/shadowcloud/index/utils/FolderDecider.scala index 7f821922..c7b35899 100644 --- a/model/src/main/scala/com/karasiq/shadowcloud/index/utils/FolderDecider.scala +++ b/model/src/main/scala/com/karasiq/shadowcloud/index/utils/FolderDecider.scala @@ -1,14 +1,12 @@ package com.karasiq.shadowcloud.index.utils - - import com.karasiq.shadowcloud.model.File import com.karasiq.shadowcloud.utils.MergeUtil.SplitDecider final case class FolderDecider(files: SplitDecider[File], folders: SplitDecider[String]) object FolderDecider { - val createWins = FolderDecider(SplitDecider.keepLeft, SplitDecider.keepLeft) - val deleteWins = FolderDecider(SplitDecider.keepRight, SplitDecider.keepRight) + val createWins = FolderDecider(SplitDecider.keepLeft, SplitDecider.keepLeft) + val deleteWins = FolderDecider(SplitDecider.keepRight, SplitDecider.keepRight) val mutualExclude = FolderDecider(SplitDecider.dropDuplicates, SplitDecider.dropDuplicates) } diff --git a/model/src/main/scala/com/karasiq/shadowcloud/index/utils/FolderDiffDecider.scala b/model/src/main/scala/com/karasiq/shadowcloud/index/utils/FolderDiffDecider.scala index faf0339c..101ff94d 100644 --- a/model/src/main/scala/com/karasiq/shadowcloud/index/utils/FolderDiffDecider.scala +++ b/model/src/main/scala/com/karasiq/shadowcloud/index/utils/FolderDiffDecider.scala @@ -6,6 +6,6 @@ final case class FolderDiffDecider(files: DiffMergeDecider[File], folders: DiffM object FolderDiffDecider { val idempotent = FolderDiffDecider(DiffMergeDecider.idempotent, DiffMergeDecider.idempotent) - val leftWins = FolderDiffDecider(DiffMergeDecider.leftWins, DiffMergeDecider.leftWins) - val rightWins = FolderDiffDecider(DiffMergeDecider.rightWins, DiffMergeDecider.rightWins) -} \ No newline at end of file + val leftWins = FolderDiffDecider(DiffMergeDecider.leftWins, DiffMergeDecider.leftWins) + val rightWins = FolderDiffDecider(DiffMergeDecider.rightWins, DiffMergeDecider.rightWins) +} diff --git a/model/src/main/scala/com/karasiq/shadowcloud/index/utils/HasEmpty.scala b/model/src/main/scala/com/karasiq/shadowcloud/index/utils/HasEmpty.scala index abc3dc9e..3290698f 100644 --- a/model/src/main/scala/com/karasiq/shadowcloud/index/utils/HasEmpty.scala +++ b/model/src/main/scala/com/karasiq/shadowcloud/index/utils/HasEmpty.scala @@ -1,7 +1,5 @@ package com.karasiq.shadowcloud.index.utils - - trait HasEmpty { def isEmpty: Boolean final def nonEmpty: Boolean = !isEmpty diff --git a/model/src/main/scala/com/karasiq/shadowcloud/index/utils/HasPath.scala b/model/src/main/scala/com/karasiq/shadowcloud/index/utils/HasPath.scala index 5a176357..8603e105 100644 --- a/model/src/main/scala/com/karasiq/shadowcloud/index/utils/HasPath.scala +++ b/model/src/main/scala/com/karasiq/shadowcloud/index/utils/HasPath.scala @@ -1,7 +1,5 @@ package com.karasiq.shadowcloud.index.utils - - import com.karasiq.shadowcloud.model.Path trait HasPath { diff --git a/model/src/main/scala/com/karasiq/shadowcloud/index/utils/HasWithoutChunks.scala b/model/src/main/scala/com/karasiq/shadowcloud/index/utils/HasWithoutChunks.scala index 2858d28e..0fccf82c 100644 --- a/model/src/main/scala/com/karasiq/shadowcloud/index/utils/HasWithoutChunks.scala +++ b/model/src/main/scala/com/karasiq/shadowcloud/index/utils/HasWithoutChunks.scala @@ -3,8 +3,7 @@ package com.karasiq.shadowcloud.index.utils trait HasWithoutChunks { type Repr - /** - * Drops data chunks from entity + /** Drops data chunks from entity * @return Entity without chunks information */ def withoutChunks: Repr diff --git a/model/src/main/scala/com/karasiq/shadowcloud/index/utils/HasWithoutData.scala b/model/src/main/scala/com/karasiq/shadowcloud/index/utils/HasWithoutData.scala index ae98f866..bd0c5d91 100644 --- a/model/src/main/scala/com/karasiq/shadowcloud/index/utils/HasWithoutData.scala +++ b/model/src/main/scala/com/karasiq/shadowcloud/index/utils/HasWithoutData.scala @@ -1,12 +1,9 @@ package com.karasiq.shadowcloud.index.utils - - trait HasWithoutData { type Repr - /** - * Drops binary data payload from entity + /** Drops binary data payload from entity * @return Entity without data */ def withoutData: Repr diff --git a/model/src/main/scala/com/karasiq/shadowcloud/index/utils/HasWithoutKeys.scala b/model/src/main/scala/com/karasiq/shadowcloud/index/utils/HasWithoutKeys.scala index 3e0cd4b7..15c69bff 100644 --- a/model/src/main/scala/com/karasiq/shadowcloud/index/utils/HasWithoutKeys.scala +++ b/model/src/main/scala/com/karasiq/shadowcloud/index/utils/HasWithoutKeys.scala @@ -3,8 +3,7 @@ package com.karasiq.shadowcloud.index.utils trait HasWithoutKeys { type Repr - /** - * Drops cryptographic keys from entity + /** Drops cryptographic keys from entity * @return Entity without keys */ def withoutKeys: Repr diff --git a/model/src/main/scala/com/karasiq/shadowcloud/index/utils/Mergeable.scala b/model/src/main/scala/com/karasiq/shadowcloud/index/utils/Mergeable.scala index 95c9c281..c4b30579 100644 --- a/model/src/main/scala/com/karasiq/shadowcloud/index/utils/Mergeable.scala +++ b/model/src/main/scala/com/karasiq/shadowcloud/index/utils/Mergeable.scala @@ -1,7 +1,5 @@ package com.karasiq.shadowcloud.index.utils - - trait Mergeable { type Repr type DiffRepr diff --git a/model/src/main/scala/com/karasiq/shadowcloud/index/utils/MergeableDiff.scala b/model/src/main/scala/com/karasiq/shadowcloud/index/utils/MergeableDiff.scala index f16a0186..865f6d62 100644 --- a/model/src/main/scala/com/karasiq/shadowcloud/index/utils/MergeableDiff.scala +++ b/model/src/main/scala/com/karasiq/shadowcloud/index/utils/MergeableDiff.scala @@ -1,7 +1,5 @@ package com.karasiq.shadowcloud.index.utils - - trait MergeableDiff extends Mergeable { type Repr type DiffRepr = Repr diff --git a/model/src/main/scala/com/karasiq/shadowcloud/model/Checksum.scala b/model/src/main/scala/com/karasiq/shadowcloud/model/Checksum.scala index b3771d0f..c0c5eed9 100644 --- a/model/src/main/scala/com/karasiq/shadowcloud/model/Checksum.scala +++ b/model/src/main/scala/com/karasiq/shadowcloud/model/Checksum.scala @@ -1,7 +1,5 @@ package com.karasiq.shadowcloud.model - - import akka.util.ByteString import com.karasiq.common.encoding.HexString import com.karasiq.common.memory.MemorySize @@ -9,9 +7,15 @@ import com.karasiq.shadowcloud.index.utils.HasEmpty import com.karasiq.shadowcloud.model.crypto.HashingMethod @SerialVersionUID(0L) -final case class Checksum(method: HashingMethod = HashingMethod.default, encMethod: HashingMethod = HashingMethod.default, - size: Long = 0, hash: ByteString = ByteString.empty, encSize: Long = 0, - encHash: ByteString = ByteString.empty) extends SCEntity with HasEmpty { +final case class Checksum( + method: HashingMethod = HashingMethod.default, + encMethod: HashingMethod = HashingMethod.default, + size: Long = 0, + hash: ByteString = ByteString.empty, + encSize: Long = 0, + encHash: ByteString = ByteString.empty +) extends SCEntity + with HasEmpty { @transient private[this] lazy val _hashCode = (size, hash).hashCode() @@ -47,7 +51,7 @@ final case class Checksum(method: HashingMethod = HashingMethod.default, encMeth } else { Seq(method.toString, encMethod.toString) } - val plain = sizeAndHash("plain", size, hash) + val plain = sizeAndHash("plain", size, hash) val encrypted = sizeAndHash("encrypted", encSize, encHash) s"Checksum(${(methods ++ Seq(plain, encrypted)).filter(_.nonEmpty).mkString(", ")})" diff --git a/model/src/main/scala/com/karasiq/shadowcloud/model/Data.scala b/model/src/main/scala/com/karasiq/shadowcloud/model/Data.scala index 20f4699b..90781f75 100644 --- a/model/src/main/scala/com/karasiq/shadowcloud/model/Data.scala +++ b/model/src/main/scala/com/karasiq/shadowcloud/model/Data.scala @@ -1,13 +1,13 @@ package com.karasiq.shadowcloud.model - - import akka.util.ByteString import com.karasiq.shadowcloud.index.utils.{HasEmpty, HasWithoutData} @SerialVersionUID(0L) final case class Data(plain: ByteString = ByteString.empty, encrypted: ByteString = ByteString.empty) - extends SCEntity with HasEmpty with HasWithoutData { + extends SCEntity + with HasEmpty + with HasWithoutData { type Repr = Data diff --git a/model/src/main/scala/com/karasiq/shadowcloud/model/File.scala b/model/src/main/scala/com/karasiq/shadowcloud/model/File.scala index 14b46285..048ea558 100644 --- a/model/src/main/scala/com/karasiq/shadowcloud/model/File.scala +++ b/model/src/main/scala/com/karasiq/shadowcloud/model/File.scala @@ -1,23 +1,30 @@ package com.karasiq.shadowcloud.model - - import com.karasiq.shadowcloud.config.SerializedProps import com.karasiq.shadowcloud.index.utils._ import com.karasiq.shadowcloud.utils.Utils @SerialVersionUID(0L) -final case class File(path: Path, id: FileId = FileId.create(), - revision: Long = 0, timestamp: Timestamp = Timestamp.now, - props: SerializedProps = SerializedProps.empty, - checksum: Checksum = Checksum.empty, chunks: Seq[Chunk] = Nil) - extends SCEntity with HasPath with HasEmpty with HasWithoutData with HasWithoutChunks with HasWithoutKeys { +final case class File( + path: Path, + id: FileId = FileId.create(), + revision: Long = 0, + timestamp: Timestamp = Timestamp.now, + props: SerializedProps = SerializedProps.empty, + checksum: Checksum = Checksum.empty, + chunks: Seq[Chunk] = Nil +) extends SCEntity + with HasPath + with HasEmpty + with HasWithoutData + with HasWithoutChunks + with HasWithoutKeys { type Repr = File require(!path.isRoot, "Root can not be a file") @transient - private[this] lazy val _hashCode = (path, id/*, revision, checksum, chunks*/).hashCode() + private[this] lazy val _hashCode = (path, id /*, revision, checksum, chunks*/ ).hashCode() def withoutData: File = { copy(chunks = chunks.map(_.withoutData)) diff --git a/model/src/main/scala/com/karasiq/shadowcloud/model/Folder.scala b/model/src/main/scala/com/karasiq/shadowcloud/model/Folder.scala index 95601067..c77cf6ce 100644 --- a/model/src/main/scala/com/karasiq/shadowcloud/model/Folder.scala +++ b/model/src/main/scala/com/karasiq/shadowcloud/model/Folder.scala @@ -7,11 +7,16 @@ import com.karasiq.shadowcloud.utils.Utils import scala.collection.GenTraversableOnce @SerialVersionUID(0L) -final case class Folder(path: Path, timestamp: Timestamp = Timestamp.now, - folders: Set[String] = Set.empty, files: Set[File] = Set.empty) - extends SCEntity with HasPath with HasEmpty with HasWithoutData with HasWithoutChunks with HasWithoutKeys with Mergeable { - - type Repr = Folder +final case class Folder(path: Path, timestamp: Timestamp = Timestamp.now, folders: Set[String] = Set.empty, files: Set[File] = Set.empty) + extends SCEntity + with HasPath + with HasEmpty + with HasWithoutData + with HasWithoutChunks + with HasWithoutKeys + with Mergeable { + + type Repr = Folder type DiffRepr = FolderDiff def addFiles(files: GenTraversableOnce[File]): Folder = { @@ -59,9 +64,7 @@ final case class Folder(path: Path, timestamp: Timestamp = Timestamp.now, } def patch(diff: FolderDiff): Folder = { - copy(path, timestamp.modified(diff.time), - folders ++ diff.newFolders -- diff.deletedFolders, - files ++ diff.newFiles -- diff.deletedFiles) + copy(path, timestamp.modified(diff.time), folders ++ diff.newFolders -- diff.deletedFolders, files ++ diff.newFiles -- diff.deletedFiles) } def withPath(newPath: Path): Folder = { @@ -93,7 +96,7 @@ final case class Folder(path: Path, timestamp: Timestamp = Timestamp.now, f.path == path && f.folders == folders && f.files == files case _ ⇒ - false + false } override def toString: String = { diff --git a/model/src/main/scala/com/karasiq/shadowcloud/model/SCEntity.scala b/model/src/main/scala/com/karasiq/shadowcloud/model/SCEntity.scala index f0db0e88..6173e00a 100644 --- a/model/src/main/scala/com/karasiq/shadowcloud/model/SCEntity.scala +++ b/model/src/main/scala/com/karasiq/shadowcloud/model/SCEntity.scala @@ -1,6 +1,5 @@ package com.karasiq.shadowcloud.model -/** - * Base serializable entity +/** Base serializable entity */ trait SCEntity diff --git a/model/src/main/scala/com/karasiq/shadowcloud/model/Timestamp.scala b/model/src/main/scala/com/karasiq/shadowcloud/model/Timestamp.scala index 1ea696c0..e3163a53 100644 --- a/model/src/main/scala/com/karasiq/shadowcloud/model/Timestamp.scala +++ b/model/src/main/scala/com/karasiq/shadowcloud/model/Timestamp.scala @@ -1,7 +1,5 @@ package com.karasiq.shadowcloud.model - - import com.karasiq.shadowcloud.utils.Utils @SerialVersionUID(0L) diff --git a/model/src/main/scala/com/karasiq/shadowcloud/model/crypto/CryptoMethod.scala b/model/src/main/scala/com/karasiq/shadowcloud/model/crypto/CryptoMethod.scala index 4208c014..c014109e 100644 --- a/model/src/main/scala/com/karasiq/shadowcloud/model/crypto/CryptoMethod.scala +++ b/model/src/main/scala/com/karasiq/shadowcloud/model/crypto/CryptoMethod.scala @@ -1,7 +1,5 @@ package com.karasiq.shadowcloud.model.crypto - - import com.karasiq.shadowcloud.config.SerializedProps import com.karasiq.shadowcloud.model.SCEntity diff --git a/model/src/main/scala/com/karasiq/shadowcloud/model/crypto/EncryptionMethod.scala b/model/src/main/scala/com/karasiq/shadowcloud/model/crypto/EncryptionMethod.scala index 57072a86..a24632a3 100644 --- a/model/src/main/scala/com/karasiq/shadowcloud/model/crypto/EncryptionMethod.scala +++ b/model/src/main/scala/com/karasiq/shadowcloud/model/crypto/EncryptionMethod.scala @@ -1,13 +1,10 @@ package com.karasiq.shadowcloud.model.crypto - - import com.karasiq.shadowcloud.config.SerializedProps @SerialVersionUID(0L) -final case class EncryptionMethod(algorithm: String, keySize: Int = 256, - config: SerializedProps = SerializedProps.empty, - provider: String = "") extends CryptoMethod { +final case class EncryptionMethod(algorithm: String, keySize: Int = 256, config: SerializedProps = SerializedProps.empty, provider: String = "") + extends CryptoMethod { @transient private[this] lazy val _hashCode = scala.util.hashing.MurmurHash3.productHash(this) @@ -26,6 +23,6 @@ final case class EncryptionMethod(algorithm: String, keySize: Int = 256, } object EncryptionMethod { - val none = EncryptionMethod("", 0) + val none = EncryptionMethod("", 0) val default = EncryptionMethod("ChaCha20", 256) } diff --git a/model/src/main/scala/com/karasiq/shadowcloud/model/crypto/EncryptionParameters.scala b/model/src/main/scala/com/karasiq/shadowcloud/model/crypto/EncryptionParameters.scala index f0157b19..218dfe0e 100644 --- a/model/src/main/scala/com/karasiq/shadowcloud/model/crypto/EncryptionParameters.scala +++ b/model/src/main/scala/com/karasiq/shadowcloud/model/crypto/EncryptionParameters.scala @@ -1,7 +1,5 @@ package com.karasiq.shadowcloud.model.crypto - - import akka.util.ByteString import com.karasiq.shadowcloud.index.utils.HasWithoutKeys @@ -11,9 +9,7 @@ sealed trait EncryptionParameters extends CryptoParameters with HasWithoutKeys { } @SerialVersionUID(0L) -final case class SymmetricEncryptionParameters(method: EncryptionMethod, - key: ByteString, - nonce: ByteString) extends EncryptionParameters { +final case class SymmetricEncryptionParameters(method: EncryptionMethod, key: ByteString, nonce: ByteString) extends EncryptionParameters { type Repr = SymmetricEncryptionParameters @@ -23,7 +19,7 @@ final case class SymmetricEncryptionParameters(method: EncryptionMethod, def isEmpty: Boolean = { key.isEmpty } - + def withoutKeys = { copy(key = ByteString.empty, nonce = ByteString.empty) } @@ -38,10 +34,9 @@ final case class SymmetricEncryptionParameters(method: EncryptionMethod, } @SerialVersionUID(0L) -final case class AsymmetricEncryptionParameters(method: EncryptionMethod, - publicKey: ByteString, - privateKey: ByteString) - extends EncryptionParameters with HasWithoutKeys { +final case class AsymmetricEncryptionParameters(method: EncryptionMethod, publicKey: ByteString, privateKey: ByteString) + extends EncryptionParameters + with HasWithoutKeys { type Repr = AsymmetricEncryptionParameters @@ -91,7 +86,7 @@ object EncryptionParameters { } def hasKeys(p: EncryptionParameters): Boolean = p match { - case SymmetricEncryptionParameters(_, key, _) ⇒ key.nonEmpty + case SymmetricEncryptionParameters(_, key, _) ⇒ key.nonEmpty case AsymmetricEncryptionParameters(_, pubKey, privKey) ⇒ pubKey.nonEmpty || privKey.nonEmpty } } diff --git a/model/src/main/scala/com/karasiq/shadowcloud/model/crypto/HashingMethod.scala b/model/src/main/scala/com/karasiq/shadowcloud/model/crypto/HashingMethod.scala index 2e0f4782..69f7b669 100644 --- a/model/src/main/scala/com/karasiq/shadowcloud/model/crypto/HashingMethod.scala +++ b/model/src/main/scala/com/karasiq/shadowcloud/model/crypto/HashingMethod.scala @@ -1,12 +1,9 @@ package com.karasiq.shadowcloud.model.crypto - - import com.karasiq.shadowcloud.config.SerializedProps @SerialVersionUID(0L) -final case class HashingMethod(algorithm: String, config: SerializedProps = SerializedProps.empty, - provider: String = "") extends CryptoMethod { +final case class HashingMethod(algorithm: String, config: SerializedProps = SerializedProps.empty, provider: String = "") extends CryptoMethod { @transient private[this] lazy val _hashCode = scala.util.hashing.MurmurHash3.productHash(this) @@ -25,6 +22,6 @@ final case class HashingMethod(algorithm: String, config: SerializedProps = Seri } object HashingMethod { - val none = HashingMethod("") + val none = HashingMethod("") val default = HashingMethod("Blake2b") } diff --git a/model/src/main/scala/com/karasiq/shadowcloud/model/crypto/SignMethod.scala b/model/src/main/scala/com/karasiq/shadowcloud/model/crypto/SignMethod.scala index ae15be3a..e4750a02 100644 --- a/model/src/main/scala/com/karasiq/shadowcloud/model/crypto/SignMethod.scala +++ b/model/src/main/scala/com/karasiq/shadowcloud/model/crypto/SignMethod.scala @@ -3,8 +3,13 @@ package com.karasiq.shadowcloud.model.crypto import com.karasiq.shadowcloud.config.SerializedProps @SerialVersionUID(0L) -final case class SignMethod(algorithm: String, hashingMethod: HashingMethod, keySize: Int = 256, - config: SerializedProps = SerializedProps.empty, provider: String = "") extends CryptoMethod { +final case class SignMethod( + algorithm: String, + hashingMethod: HashingMethod, + keySize: Int = 256, + config: SerializedProps = SerializedProps.empty, + provider: String = "" +) extends CryptoMethod { @transient private[this] lazy val _hashCode = scala.util.hashing.MurmurHash3.productHash(this) @@ -17,7 +22,8 @@ final case class SignMethod(algorithm: String, hashingMethod: HashingMethod, key if (CryptoMethod.isNoOpMethod(this)) { "SignMethod.none" } else { - s"SignMethod(${if (provider.isEmpty) algorithm else provider + ":" + algorithm}, $hashingMethod, $keySize bits${if (config.isEmpty) "" else ", " + config})" + s"SignMethod(${if (provider.isEmpty) algorithm else provider + ":" + algorithm}, $hashingMethod, $keySize bits${if (config.isEmpty) "" + else ", " + config})" } } } diff --git a/model/src/main/scala/com/karasiq/shadowcloud/model/crypto/SignParameters.scala b/model/src/main/scala/com/karasiq/shadowcloud/model/crypto/SignParameters.scala index 8fc7ffac..4348b94f 100644 --- a/model/src/main/scala/com/karasiq/shadowcloud/model/crypto/SignParameters.scala +++ b/model/src/main/scala/com/karasiq/shadowcloud/model/crypto/SignParameters.scala @@ -1,7 +1,5 @@ package com.karasiq.shadowcloud.model.crypto - - import akka.util.ByteString import com.karasiq.shadowcloud.index.utils.HasWithoutKeys diff --git a/model/src/main/scala/com/karasiq/shadowcloud/model/keys/KeyChain.scala b/model/src/main/scala/com/karasiq/shadowcloud/model/keys/KeyChain.scala index 08cbcda3..88fabf8d 100644 --- a/model/src/main/scala/com/karasiq/shadowcloud/model/keys/KeyChain.scala +++ b/model/src/main/scala/com/karasiq/shadowcloud/model/keys/KeyChain.scala @@ -5,8 +5,7 @@ import com.karasiq.shadowcloud.model.{RegionId, SCEntity} import com.karasiq.shadowcloud.model.keys.KeyProps.RegionSet @SerialVersionUID(1L) -final case class KeyChain(keys: Seq[KeyProps]) - extends SCEntity with HasEmpty with HasWithoutKeys { +final case class KeyChain(keys: Seq[KeyProps]) extends SCEntity with HasEmpty with HasWithoutKeys { type Repr = KeyChain def isEmpty: Boolean = keys.isEmpty diff --git a/model/src/main/scala/com/karasiq/shadowcloud/model/keys/KeyProps.scala b/model/src/main/scala/com/karasiq/shadowcloud/model/keys/KeyProps.scala index 5e4b367a..a3b0be7f 100644 --- a/model/src/main/scala/com/karasiq/shadowcloud/model/keys/KeyProps.scala +++ b/model/src/main/scala/com/karasiq/shadowcloud/model/keys/KeyProps.scala @@ -13,4 +13,4 @@ object KeyProps { def enabledOn(regionSet: RegionSet, regionId: RegionId) = regionSet.isEmpty || regionSet.contains(regionId) } -} \ No newline at end of file +} diff --git a/model/src/main/scala/com/karasiq/shadowcloud/model/keys/KeySet.scala b/model/src/main/scala/com/karasiq/shadowcloud/model/keys/KeySet.scala index acd9d944..8ccdbef3 100644 --- a/model/src/main/scala/com/karasiq/shadowcloud/model/keys/KeySet.scala +++ b/model/src/main/scala/com/karasiq/shadowcloud/model/keys/KeySet.scala @@ -5,15 +5,14 @@ import com.karasiq.shadowcloud.model.SCEntity import com.karasiq.shadowcloud.model.crypto.{EncryptionParameters, SignParameters} @SerialVersionUID(0L) -final case class KeySet(id: KeyId, encryption: EncryptionParameters, signing: SignParameters) - extends SCEntity with HasEmpty with HasWithoutKeys { +final case class KeySet(id: KeyId, encryption: EncryptionParameters, signing: SignParameters) extends SCEntity with HasEmpty with HasWithoutKeys { type Repr = KeySet - + def isEmpty: Boolean = encryption.isEmpty && encryption.isEmpty - def withoutKeys = copy(signing = signing.withoutKeys, encryption = encryption.withoutKeys) + def withoutKeys = copy(signing = signing.withoutKeys, encryption = encryption.withoutKeys) } object KeySet { - val empty = KeySet(KeyId.empty, EncryptionParameters.empty, SignParameters.empty) -} \ No newline at end of file + val empty = KeySet(KeyId.empty, EncryptionParameters.empty, SignParameters.empty) +} diff --git a/model/src/main/scala/com/karasiq/shadowcloud/model/utils/FileAvailability.scala b/model/src/main/scala/com/karasiq/shadowcloud/model/utils/FileAvailability.scala index 4a768664..bfa8c6b9 100644 --- a/model/src/main/scala/com/karasiq/shadowcloud/model/utils/FileAvailability.scala +++ b/model/src/main/scala/com/karasiq/shadowcloud/model/utils/FileAvailability.scala @@ -9,7 +9,7 @@ final case class FileAvailability(file: File, chunksByStorage: Map[StorageId, Se def totalPercentage: Double = { if (isEmpty) return 0 - val totalChunks = file.chunks.length + val totalChunks = file.chunks.length val availableChunks = chunksByStorage.values.flatten.toSet availableChunks.size.toDouble / totalChunks * 100 } @@ -22,7 +22,7 @@ final case class FileAvailability(file: File, chunksByStorage: Map[StorageId, Se } def isFullyAvailable: Boolean = { - val fileChunks = file.chunks.toSet + val fileChunks = file.chunks.toSet val actualChunks = chunksByStorage.flatMap(_._2).toSet fileChunks == actualChunks } diff --git a/model/src/main/scala/com/karasiq/shadowcloud/model/utils/GCReport.scala b/model/src/main/scala/com/karasiq/shadowcloud/model/utils/GCReport.scala index f5a1a4cc..186de3e3 100644 --- a/model/src/main/scala/com/karasiq/shadowcloud/model/utils/GCReport.scala +++ b/model/src/main/scala/com/karasiq/shadowcloud/model/utils/GCReport.scala @@ -13,14 +13,13 @@ final case class GCReport(regionId: RegionId, regionState: RegionGCState, storag object GCReport { @SerialVersionUID(0L) - final case class RegionGCState(orphanedChunks: Set[Chunk], - oldFiles: Set[File], - expiredMetadata: Set[FileId]) extends HasEmpty { + final case class RegionGCState(orphanedChunks: Set[Chunk], oldFiles: Set[File], expiredMetadata: Set[FileId]) extends HasEmpty { def isEmpty: Boolean = orphanedChunks.isEmpty && oldFiles.isEmpty && expiredMetadata.isEmpty override def toString: String = { - s"RegionGCState(orphaned chunks = [${Utils.printChunkHashes(orphanedChunks)}], old files = [${Utils.printValues(oldFiles)}], expired metadata = [${Utils.printValues(expiredMetadata)}])" + s"RegionGCState(orphaned chunks = [${Utils + .printChunkHashes(orphanedChunks)}], old files = [${Utils.printValues(oldFiles)}], expired metadata = [${Utils.printValues(expiredMetadata)}])" } } diff --git a/model/src/main/scala/com/karasiq/shadowcloud/model/utils/HealthStatus.scala b/model/src/main/scala/com/karasiq/shadowcloud/model/utils/HealthStatus.scala index fc53d098..9b64c86c 100644 --- a/model/src/main/scala/com/karasiq/shadowcloud/model/utils/HealthStatus.scala +++ b/model/src/main/scala/com/karasiq/shadowcloud/model/utils/HealthStatus.scala @@ -19,4 +19,4 @@ object HealthStatus { val percentage = Utils.percents(hs.usedSpace, hs.totalSpace).toInt math.max(0, math.min(100, percentage)) } -} \ No newline at end of file +} diff --git a/model/src/main/scala/com/karasiq/shadowcloud/model/utils/IndexScope.scala b/model/src/main/scala/com/karasiq/shadowcloud/model/utils/IndexScope.scala index ac062ce7..706f6ce3 100644 --- a/model/src/main/scala/com/karasiq/shadowcloud/model/utils/IndexScope.scala +++ b/model/src/main/scala/com/karasiq/shadowcloud/model/utils/IndexScope.scala @@ -5,11 +5,11 @@ import com.karasiq.shadowcloud.model.SequenceNr sealed trait IndexScope object IndexScope { - case object Current extends IndexScope - case object Persisted extends IndexScope + case object Current extends IndexScope + case object Persisted extends IndexScope final case class UntilSequenceNr(sequenceNr: SequenceNr) extends IndexScope - final case class UntilTime(timestamp: Long) extends IndexScope + final case class UntilTime(timestamp: Long) extends IndexScope @inline def default: IndexScope = Current -} \ No newline at end of file +} diff --git a/model/src/main/scala/com/karasiq/shadowcloud/model/utils/RegionStateReport.scala b/model/src/main/scala/com/karasiq/shadowcloud/model/utils/RegionStateReport.scala index e6c7f812..84bc8912 100644 --- a/model/src/main/scala/com/karasiq/shadowcloud/model/utils/RegionStateReport.scala +++ b/model/src/main/scala/com/karasiq/shadowcloud/model/utils/RegionStateReport.scala @@ -9,12 +9,20 @@ final case class RegionStateReport(regions: Map[RegionId, RegionStatus], storage object RegionStateReport { @SerialVersionUID(0L) - final case class RegionStatus(regionId: RegionId, regionConfig: SerializedProps = SerializedProps.empty, - suspended: Boolean = false, storages: Set[String] = Set.empty) + final case class RegionStatus( + regionId: RegionId, + regionConfig: SerializedProps = SerializedProps.empty, + suspended: Boolean = false, + storages: Set[String] = Set.empty + ) @SerialVersionUID(0L) - final case class StorageStatus(storageId: StorageId, storageProps: SerializedProps = SerializedProps.empty, - suspended: Boolean = false, regions: Set[String] = Set.empty) + final case class StorageStatus( + storageId: StorageId, + storageProps: SerializedProps = SerializedProps.empty, + suspended: Boolean = false, + regions: Set[String] = Set.empty + ) val empty = new RegionStateReport(Map.empty, Map.empty) } diff --git a/model/src/main/scala/com/karasiq/shadowcloud/model/utils/SyncReport.scala b/model/src/main/scala/com/karasiq/shadowcloud/model/utils/SyncReport.scala index 1824ed5a..d4a7a9f0 100644 --- a/model/src/main/scala/com/karasiq/shadowcloud/model/utils/SyncReport.scala +++ b/model/src/main/scala/com/karasiq/shadowcloud/model/utils/SyncReport.scala @@ -6,9 +6,11 @@ import com.karasiq.shadowcloud.model.SequenceNr import com.karasiq.shadowcloud.utils.Utils @SerialVersionUID(0L) -final case class SyncReport(read: Map[SequenceNr, IndexDiff] = Map.empty, - written: Map[SequenceNr, IndexDiff] = Map.empty, - deleted: Set[SequenceNr] = Set.empty) extends HasEmpty { +final case class SyncReport( + read: Map[SequenceNr, IndexDiff] = Map.empty, + written: Map[SequenceNr, IndexDiff] = Map.empty, + deleted: Set[SequenceNr] = Set.empty +) extends HasEmpty { def isEmpty: Boolean = read.isEmpty && written.isEmpty && deleted.isEmpty override def toString: String = { diff --git a/model/src/main/scala/com/karasiq/shadowcloud/storage/StorageIOResult.scala b/model/src/main/scala/com/karasiq/shadowcloud/storage/StorageIOResult.scala index b1d22bdd..9d9f7105 100644 --- a/model/src/main/scala/com/karasiq/shadowcloud/storage/StorageIOResult.scala +++ b/model/src/main/scala/com/karasiq/shadowcloud/storage/StorageIOResult.scala @@ -1,7 +1,5 @@ package com.karasiq.shadowcloud.storage - - import com.karasiq.shadowcloud.exceptions.StorageException import com.karasiq.shadowcloud.model.Path @@ -18,7 +16,7 @@ object StorageIOResult { } final case class Failure(path: Path, error: StorageException) extends StorageIOResult { - val count: Long = 0L + val count: Long = 0L val isSuccess: Boolean = false } diff --git a/model/src/main/scala/com/karasiq/shadowcloud/utils/MergeUtil.scala b/model/src/main/scala/com/karasiq/shadowcloud/utils/MergeUtil.scala index 93063308..4a06288e 100644 --- a/model/src/main/scala/com/karasiq/shadowcloud/utils/MergeUtil.scala +++ b/model/src/main/scala/com/karasiq/shadowcloud/utils/MergeUtil.scala @@ -2,7 +2,6 @@ package com.karasiq.shadowcloud.utils import scala.collection.mutable - object MergeUtil { // ----------------------------------------------------------------------- // Types @@ -18,18 +17,18 @@ object MergeUtil { def right: T } - final case class Left[+T](value: T) extends State[T] with SingleSide[T] + final case class Left[+T](value: T) extends State[T] with SingleSide[T] final case class Right[+T](value: T) extends State[T] with SingleSide[T] final case class Equal[+T](value: T) extends State[T] with TwoSides[T] { - def left = value + def left = value def right = value } final case class Conflict[+T](left: T, right: T) extends State[T] with TwoSides[T] } - + import State.{Conflict, Equal, Left, Right} - type Decider[V] = PartialFunction[State[V], Option[V]] + type Decider[V] = PartialFunction[State[V], Option[V]] type SplitDecider[V] = Equal[V] ⇒ Option[State[V]] // ----------------------------------------------------------------------- @@ -77,7 +76,6 @@ object MergeUtil { compareSets(left, right).flatMap(func.orElse(Decider.default).apply(_)) } - // ----------------------------------------------------------------------- // Split // ----------------------------------------------------------------------- @@ -126,9 +124,9 @@ object MergeUtil { Some(right) } - def diff[V]: Decider[V] = { case Equal(_) ⇒ None } - def dropLeft[V]: Decider[V] = { case Left(_) ⇒ None } - def dropRight[V]: Decider[V] = { case Right(_) ⇒ None } + def diff[V]: Decider[V] = { case Equal(_) ⇒ None } + def dropLeft[V]: Decider[V] = { case Left(_) ⇒ None } + def dropRight[V]: Decider[V] = { case Right(_) ⇒ None } def dropUnique[V]: Decider[V] = { case Right(_) | Left(_) ⇒ None } def intersect[V]: Decider[V] = { @@ -161,8 +159,8 @@ object MergeUtil { // ----------------------------------------------------------------------- object SplitDecider { def dropDuplicates[V]: SplitDecider[V] = _ ⇒ None - def keepBoth[V]: SplitDecider[V] = Some.apply - def keepLeft[V]: SplitDecider[V] = state ⇒ Some(Left(state.value)) - def keepRight[V]: SplitDecider[V] = state ⇒ Some(Right(state.value)) + def keepBoth[V]: SplitDecider[V] = Some.apply + def keepLeft[V]: SplitDecider[V] = state ⇒ Some(Left(state.value)) + def keepRight[V]: SplitDecider[V] = state ⇒ Some(Right(state.value)) } } diff --git a/model/src/main/scala/com/karasiq/shadowcloud/utils/ProtobufUtils.scala b/model/src/main/scala/com/karasiq/shadowcloud/utils/ProtobufUtils.scala index 9e40644e..bc68514a 100644 --- a/model/src/main/scala/com/karasiq/shadowcloud/utils/ProtobufUtils.scala +++ b/model/src/main/scala/com/karasiq/shadowcloud/utils/ProtobufUtils.scala @@ -4,7 +4,7 @@ import java.nio.ByteBuffer import java.util.UUID import akka.util.ByteString -import com.google.protobuf.{ByteString => PBByteString} +import com.google.protobuf.{ByteString ⇒ PBByteString} import com.karasiq.shadowcloud.model.Path import scalapb.TypeMapper @@ -16,7 +16,7 @@ object ProtobufUtils { } { bs ⇒ if (bs.isEmpty) PBByteString.EMPTY else PBByteString.copyFrom(bs.toArray) } - + implicit val uuidMapper = TypeMapper[PBByteString, UUID] { bs ⇒ if (bs.isEmpty) { emptyUUID diff --git a/persistence/src/main/scala/com/karasiq/shadowcloud/persistence/h2/H2AkkaJournal.scala b/persistence/src/main/scala/com/karasiq/shadowcloud/persistence/h2/H2AkkaJournal.scala index 0260eba1..c554d25f 100644 --- a/persistence/src/main/scala/com/karasiq/shadowcloud/persistence/h2/H2AkkaJournal.scala +++ b/persistence/src/main/scala/com/karasiq/shadowcloud/persistence/h2/H2AkkaJournal.scala @@ -16,11 +16,11 @@ final class H2AkkaJournal extends AsyncWriteJournal { // ----------------------------------------------------------------------- // Context // ----------------------------------------------------------------------- - private[this] val h2DB = H2DB(context.system) + private[this] val h2DB = H2DB(context.system) private[this] implicit val materializer = Materializer.matFromSystem(context.system) import context.dispatcher - import h2DB.context.{run => runQuery, _} + import h2DB.context.{run ⇒ runQuery, _} // ----------------------------------------------------------------------- // Schema @@ -29,11 +29,12 @@ final class H2AkkaJournal extends AsyncWriteJournal { @SerialVersionUID(0L) final case class DBMessage(persistenceId: String, sequenceNr: Long, ordering: Long, tags: Set[String], message: ByteString) - implicit val tagsEncoder: Encoder[Set[String]] = encoder(java.sql.Types.ARRAY, (index, value, row) ⇒ - row.setObject(index, value.toArray, java.sql.Types.ARRAY)) + implicit val tagsEncoder: Encoder[Set[String]] = + encoder(java.sql.Types.ARRAY, (index, value, row) ⇒ row.setObject(index, value.toArray, java.sql.Types.ARRAY)) - implicit val tagsDecoder: Decoder[Set[String]] = decoder(java.sql.Types.ARRAY, (index, row) ⇒ - row.getArray(index).getArray().asInstanceOf[Array[java.lang.Object]].toSet.asInstanceOf[Set[String]] + implicit val tagsDecoder: Decoder[Set[String]] = decoder( + java.sql.Types.ARRAY, + (index, row) ⇒ row.getArray(index).getArray().asInstanceOf[Array[java.lang.Object]].toSet.asInstanceOf[Set[String]] ) implicit val journalRowSchemaMeta = schemaMeta[DBMessage]("sc_akka_journal") @@ -110,11 +111,13 @@ final class H2AkkaJournal extends AsyncWriteJournal { Future.fromTry(Try(runQuery(query))) } - def asyncReplayMessages(persistenceId: String, fromSequenceNr: Long, toSequenceNr: Long, max: Long) - (recoveryCallback: (PersistentRepr) ⇒ Unit): Future[Unit] = { + def asyncReplayMessages(persistenceId: String, fromSequenceNr: Long, toSequenceNr: Long, max: Long)( + recoveryCallback: (PersistentRepr) ⇒ Unit + ): Future[Unit] = { val maxInt = Math.min(max, Int.MaxValue).toInt val query = quote { - queries.messagesForPersistenceId(persistenceId, fromSequenceNr, toSequenceNr) + queries + .messagesForPersistenceId(persistenceId, fromSequenceNr, toSequenceNr) .take(lift(maxInt)) .map(_.message) } diff --git a/persistence/src/main/scala/com/karasiq/shadowcloud/persistence/h2/H2AkkaSnapshotStore.scala b/persistence/src/main/scala/com/karasiq/shadowcloud/persistence/h2/H2AkkaSnapshotStore.scala index 1e1745d9..d0068250 100644 --- a/persistence/src/main/scala/com/karasiq/shadowcloud/persistence/h2/H2AkkaSnapshotStore.scala +++ b/persistence/src/main/scala/com/karasiq/shadowcloud/persistence/h2/H2AkkaSnapshotStore.scala @@ -15,7 +15,7 @@ final class H2AkkaSnapshotStore extends SnapshotStore { // Context // ----------------------------------------------------------------------- private[this] val h2db = H2DB(context.system) - import h2db.context.{run => runQuery, _} + import h2db.context.{run ⇒ runQuery, _} // ----------------------------------------------------------------------- // Schema @@ -33,11 +33,12 @@ final class H2AkkaSnapshotStore extends SnapshotStore { .filter(s ⇒ s.persistenceId == lift(persistenceId) && s.sequenceNr == lift(sequenceNr)) } - def snapshotsByCriteria(persistenceId: String, minSequenceNr: Long, maxSequenceNr: Long, - minTimestamp: Long, maxTimestamp: Long) = quote { - query[DBSnapshot].filter(s ⇒ s.persistenceId == lift(persistenceId) && - s.sequenceNr >= lift(minSequenceNr) && s.sequenceNr <= lift(maxSequenceNr) && - s.timestamp >= lift(minTimestamp) && s.timestamp <= lift(maxTimestamp)) + def snapshotsByCriteria(persistenceId: String, minSequenceNr: Long, maxSequenceNr: Long, minTimestamp: Long, maxTimestamp: Long) = quote { + query[DBSnapshot].filter(s ⇒ + s.persistenceId == lift(persistenceId) && + s.sequenceNr >= lift(minSequenceNr) && s.sequenceNr <= lift(maxSequenceNr) && + s.timestamp >= lift(minTimestamp) && s.timestamp <= lift(maxTimestamp) + ) } def saveSnapshot(data: DBSnapshot) = quote { @@ -61,8 +62,7 @@ final class H2AkkaSnapshotStore extends SnapshotStore { } def toSelectedSnapshot(snapshot: DBSnapshot): SelectedSnapshot = { - SelectedSnapshot(SnapshotMetadata(snapshot.persistenceId, snapshot.sequenceNr, - snapshot.timestamp), deserializeSnapshot(snapshot.snapshot)) + SelectedSnapshot(SnapshotMetadata(snapshot.persistenceId, snapshot.sequenceNr, snapshot.timestamp), deserializeSnapshot(snapshot.snapshot)) } def toDBSnapshot(metadata: SnapshotMetadata, data: Any): DBSnapshot = { @@ -75,8 +75,8 @@ final class H2AkkaSnapshotStore extends SnapshotStore { // ----------------------------------------------------------------------- def loadAsync(persistenceId: String, criteria: SnapshotSelectionCriteria): Future[Option[SelectedSnapshot]] = { val query = quote { - queries.snapshotsByCriteria(persistenceId, criteria.minSequenceNr, - criteria.maxSequenceNr, criteria.minTimestamp, criteria.maxTimestamp) + queries + .snapshotsByCriteria(persistenceId, criteria.minSequenceNr, criteria.maxSequenceNr, criteria.minTimestamp, criteria.maxTimestamp) .sortBy(_.sequenceNr)(Ord.desc) .take(1) } @@ -94,8 +94,9 @@ final class H2AkkaSnapshotStore extends SnapshotStore { } def deleteAsync(persistenceId: String, criteria: SnapshotSelectionCriteria): Future[Unit] = { - val query = quote(queries.snapshotsByCriteria(persistenceId, criteria.minSequenceNr, - criteria.maxSequenceNr, criteria.minTimestamp, criteria.maxTimestamp).delete) + val query = quote( + queries.snapshotsByCriteria(persistenceId, criteria.minSequenceNr, criteria.maxSequenceNr, criteria.minTimestamp, criteria.maxTimestamp).delete + ) Future.fromTry(Try(runQuery(query))) } } diff --git a/persistence/src/main/scala/com/karasiq/shadowcloud/persistence/h2/H2Context.scala b/persistence/src/main/scala/com/karasiq/shadowcloud/persistence/h2/H2Context.scala index 579c7be1..1ae70a26 100644 --- a/persistence/src/main/scala/com/karasiq/shadowcloud/persistence/h2/H2Context.scala +++ b/persistence/src/main/scala/com/karasiq/shadowcloud/persistence/h2/H2Context.scala @@ -11,8 +11,9 @@ object H2Context { def apply(config: Config, password: String): ContextT = { try { new ContextT(createJdbcConfig(config, password)) - } catch { case NonFatal(err) => - throw new RuntimeException("Failed to open DB", err) + } catch { + case NonFatal(err) ⇒ + throw new RuntimeException("Failed to open DB", err) } } @@ -21,19 +22,21 @@ object H2Context { import com.karasiq.common.configs.ConfigImplicits._ require(!password.contains(" "), "Space character is not allowed") - val path = config.getString("path") - val compress = config.withDefault(true, _.getBoolean("compress")) - val cipher = config.optional(_.getString("cipher")).filter(_.nonEmpty) + val path = config.getString("path") + val compress = config.withDefault(true, _.getBoolean("compress")) + val cipher = config.optional(_.getString("cipher")).filter(_.nonEmpty) val initScript = config.optional(_.getString("init-script")).filter(_.nonEmpty) //noinspection SpellCheckingInspection - ConfigFactory.parseMap(Map( - "dataSourceClassName" → "org.h2.jdbcx.JdbcDataSource", - "dataSource.url" → (s"jdbc:h2:file:$path;COMPRESS=$compress;DB_CLOSE_ON_EXIT=FALSE" - + cipher.fold("")(cipher ⇒ s";CIPHER=$cipher") - + initScript.fold("")(script ⇒ s";INIT=RUNSCRIPT FROM '$script'")), - "dataSource.user" → "sa", - "dataSource.password" → (if (cipher.nonEmpty) s"$password sa" else "sa") - ).asJava) + ConfigFactory.parseMap( + Map( + "dataSourceClassName" → "org.h2.jdbcx.JdbcDataSource", + "dataSource.url" → (s"jdbc:h2:file:$path;COMPRESS=$compress;DB_CLOSE_ON_EXIT=FALSE" + + cipher.fold("")(cipher ⇒ s";CIPHER=$cipher") + + initScript.fold("")(script ⇒ s";INIT=RUNSCRIPT FROM '$script'")), + "dataSource.user" → "sa", + "dataSource.password" → (if (cipher.nonEmpty) s"$password sa" else "sa") + ).asJava + ) } } diff --git a/persistence/src/main/scala/com/karasiq/shadowcloud/persistence/h2/H2KeyProvider.scala b/persistence/src/main/scala/com/karasiq/shadowcloud/persistence/h2/H2KeyProvider.scala index f6b558c2..d582c0ca 100644 --- a/persistence/src/main/scala/com/karasiq/shadowcloud/persistence/h2/H2KeyProvider.scala +++ b/persistence/src/main/scala/com/karasiq/shadowcloud/persistence/h2/H2KeyProvider.scala @@ -28,17 +28,17 @@ final class H2KeyProvider(actorSystem: ActorSystem) extends KeyProvider { @SerialVersionUID(0L) final case class DBKey(id: UUID, regionSet: RegionSet, forEncryption: Boolean, forDecryption: Boolean, key: ByteString) - implicit val regionsEncoder: Encoder[RegionSet] = encoder(java.sql.Types.ARRAY, (index, value, row) ⇒ - row.setObject(index, value.toArray, java.sql.Types.ARRAY)) + implicit val regionsEncoder: Encoder[RegionSet] = + encoder(java.sql.Types.ARRAY, (index, value, row) ⇒ row.setObject(index, value.toArray, java.sql.Types.ARRAY)) - implicit val regionsDecoder: Decoder[RegionSet] = decoder(java.sql.Types.ARRAY, (index, row) ⇒ - row.getArray(index).getArray().asInstanceOf[Array[java.lang.Object]].toSet.asInstanceOf[Set[String]] + implicit val regionsDecoder: Decoder[RegionSet] = decoder( + java.sql.Types.ARRAY, + (index, row) ⇒ row.getArray(index).getArray().asInstanceOf[Array[java.lang.Object]].toSet.asInstanceOf[Set[String]] ) //noinspection TypeAnnotation - implicit val keySchemaMeta = schemaMeta[DBKey]("sc_keys", _.id → "key_id", - _.forEncryption → "for_encryption", _.forDecryption → "for_decryption", - _.key → "serialized_key") + implicit val keySchemaMeta = + schemaMeta[DBKey]("sc_keys", _.id → "key_id", _.forEncryption → "for_encryption", _.forDecryption → "for_decryption", _.key → "serialized_key") } import schema._ diff --git a/project/ProjectDeps.scala b/project/ProjectDeps.scala index 6cd07203..ca402bbe 100644 --- a/project/ProjectDeps.scala +++ b/project/ProjectDeps.scala @@ -79,8 +79,8 @@ object ProjectDeps { // http://h2database.com/html/main.html def h2: Deps = Seq( - "com.h2database" % "h2" % "1.4.192", - "io.getquill" %% "quill-jdbc" % "1.2.1" + "com.h2database" % "h2" % "1.4.192", + "io.getquill" %% "quill-jdbc" % "1.2.1" ) // https://tika.apache.org/ @@ -207,10 +207,10 @@ object ProjectDeps { def webzinc: Deps = { val version = "1.0.10" Seq( - "net.sourceforge.htmlunit" % "htmlunit" % "2.40.0", - "com.github.karasiq" %% "commons-network" % "1.0.10", - "com.github.karasiq" %% "webzinc" % version, - "com.github.karasiq" %% "webzinc-htmlunit" % version + "net.sourceforge.htmlunit" % "htmlunit" % "2.40.0", + "com.github.karasiq" %% "commons-network" % "1.0.10", + "com.github.karasiq" %% "webzinc" % version, + "com.github.karasiq" %% "webzinc-htmlunit" % version ) } diff --git a/project/ScalaJSDeps.scala b/project/ScalaJSDeps.scala index 2b04cd44..9368d4f5 100644 --- a/project/ScalaJSDeps.scala +++ b/project/ScalaJSDeps.scala @@ -9,14 +9,14 @@ object ScalaJSDeps { def actors: Deps = { libraryDependencies ++= Seq( - "org.akka-js" %%% "akkajsactor" % version, + "org.akka-js" %%% "akkajsactor" % version, "org.akka-js" %%% "akkajstestkit" % version % "test" ) } def streams: Deps = { libraryDependencies ++= Seq( - "org.akka-js" %%% "akkajsactorstream" % version, + "org.akka-js" %%% "akkajsactorstream" % version, "org.akka-js" %%% "akkajsstreamtestkit" % version % "test" ) } @@ -28,9 +28,9 @@ object ScalaJSDeps { def bootstrap: Deps = { libraryDependencies ++= Seq( - "com.lihaoyi" %%% "scalatags" % "0.6.7", - "com.github.karasiq" %%% "scalajs-bootstrap" % "2.3.4", - "org.querki" %%% "bootstrap-datepicker-facade" % "0.9" + "com.lihaoyi" %%% "scalatags" % "0.6.7", + "com.github.karasiq" %%% "scalajs-bootstrap" % "2.3.4", + "org.querki" %%% "bootstrap-datepicker-facade" % "0.9" ) } diff --git a/project/WebDeps.scala b/project/WebDeps.scala index dc265655..23dac963 100644 --- a/project/WebDeps.scala +++ b/project/WebDeps.scala @@ -92,7 +92,7 @@ object WebDeps { val videoJs = github("videojs", "video.js", "v5.8.0") / "dist" Seq( Script from url(videoJs % "video.min.js"), - Style from url(videoJs % "video-js.min.css") + Style from url(videoJs % "video-js.min.css") ) } @@ -134,8 +134,8 @@ object WebDeps { val highlightJSStyle = "github" - val markedJS = "org.webjars.bower" % "marked" % "0.3.5" - val highlightJS = "org.webjars" % "highlightjs" % "9.2.0" + val markedJS = "org.webjars.bower" % "marked" % "0.3.5" + val highlightJS = "org.webjars" % "highlightjs" % "9.2.0" val tabOverrideJS = github("wjbryant", "taboverride", "4.0.3") / "build" / "output" val scripts = Seq( diff --git a/project/plugins.sbt b/project/plugins.sbt index 989147a0..3bb15051 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -10,7 +10,7 @@ addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.7.0") addSbtPlugin("org.scala-js" % "sbt-scalajs" % "0.6.32") -addSbtPlugin("org.portable-scala" % "sbt-scalajs-crossproject" % "0.6.1") +addSbtPlugin("org.portable-scala" % "sbt-scalajs-crossproject" % "0.6.1") addSbtPlugin("org.portable-scala" % "sbt-scala-native-crossproject" % "0.6.1") diff --git a/project/webjars.sbt b/project/webjars.sbt index f4911e62..599eb1d1 100644 --- a/project/webjars.sbt +++ b/project/webjars.sbt @@ -1,5 +1,5 @@ libraryDependencies ++= Seq( - "com.lihaoyi" %% "scalatags" % "0.6.7", + "com.lihaoyi" %% "scalatags" % "0.6.7", "org.webjars.bower" % "marked" % "0.3.5", "org.webjars" % "highlightjs" % "9.2.0", "org.webjars" % "dropzone" % "5.5.0", diff --git a/serialization/.jvm/src/main/scala/com/karasiq/shadowcloud/serialization/kryo/GeneratedMessageSerializer.scala b/serialization/.jvm/src/main/scala/com/karasiq/shadowcloud/serialization/kryo/GeneratedMessageSerializer.scala index afdb94f2..4d44856a 100644 --- a/serialization/.jvm/src/main/scala/com/karasiq/shadowcloud/serialization/kryo/GeneratedMessageSerializer.scala +++ b/serialization/.jvm/src/main/scala/com/karasiq/shadowcloud/serialization/kryo/GeneratedMessageSerializer.scala @@ -8,11 +8,10 @@ import com.twitter.chill import com.karasiq.shadowcloud.serialization.internal.CompanionReflectSerializer // ScalaPB serializer -private[kryo] final class GeneratedMessageSerializer - extends chill.KSerializer[GeneratedMessage](false, true) with CompanionReflectSerializer { +private[kryo] final class GeneratedMessageSerializer extends chill.KSerializer[GeneratedMessage](false, true) with CompanionReflectSerializer { def read(kryo: Kryo, input: Input, `type`: Class[GeneratedMessage]): GeneratedMessage = { - val bytes = kryo.readObject(input, classOf[Array[Byte]]) + val bytes = kryo.readObject(input, classOf[Array[Byte]]) val companion = getCompanion[GeneratedMessageCompanion[_ <: GeneratedMessage]](`type`) companion.parseFrom(bytes) } diff --git a/serialization/.jvm/src/main/scala/com/karasiq/shadowcloud/serialization/kryo/SCKryoRegistrar.scala b/serialization/.jvm/src/main/scala/com/karasiq/shadowcloud/serialization/kryo/SCKryoRegistrar.scala index ed83310a..53bed8a5 100644 --- a/serialization/.jvm/src/main/scala/com/karasiq/shadowcloud/serialization/kryo/SCKryoRegistrar.scala +++ b/serialization/.jvm/src/main/scala/com/karasiq/shadowcloud/serialization/kryo/SCKryoRegistrar.scala @@ -20,14 +20,36 @@ private[kryo] final class SCKryoRegistrar extends IKryoRegistrar { register(kryo, new WrappedConfigSerializer) register(kryo, new GeneratedMessageSerializer) register(kryo, new TimestampSerializer) - kryo.registerClasses(Iterator(classOf[Checksum], classOf[Chunk], classOf[ChunkIndex], classOf[Timestamp], - classOf[FolderIndex], classOf[ChunkIndexDiff], classOf[Data], classOf[File], classOf[Folder], classOf[FolderDiff], - classOf[FolderIndexDiff], classOf[IndexDiff], classOf[Path], classOf[SerializedProps], classOf[HashingMethod], classOf[EncryptionMethod], - classOf[SignMethod], classOf[SymmetricEncryptionParameters], classOf[AsymmetricEncryptionParameters], classOf[SignParameters], classOf[KeySet], - classOf[KeyChain], classOf[KeyProps], classOf[IndexData] - )) + kryo.registerClasses( + Iterator( + classOf[Checksum], + classOf[Chunk], + classOf[ChunkIndex], + classOf[Timestamp], + classOf[FolderIndex], + classOf[ChunkIndexDiff], + classOf[Data], + classOf[File], + classOf[Folder], + classOf[FolderDiff], + classOf[FolderIndexDiff], + classOf[IndexDiff], + classOf[Path], + classOf[SerializedProps], + classOf[HashingMethod], + classOf[EncryptionMethod], + classOf[SignMethod], + classOf[SymmetricEncryptionParameters], + classOf[AsymmetricEncryptionParameters], + classOf[SignParameters], + classOf[KeySet], + classOf[KeyChain], + classOf[KeyProps], + classOf[IndexData] + ) + ) } - + @inline private[this] def register[T: ClassTag](kryo: Kryo, serializer: chill.KSerializer[T]): Unit = { if (!kryo.alreadyRegistered[T]) { diff --git a/serialization/.jvm/src/main/scala/com/karasiq/shadowcloud/serialization/kryo/TimestampSerializer.scala b/serialization/.jvm/src/main/scala/com/karasiq/shadowcloud/serialization/kryo/TimestampSerializer.scala index d16d0a30..b3f355b1 100644 --- a/serialization/.jvm/src/main/scala/com/karasiq/shadowcloud/serialization/kryo/TimestampSerializer.scala +++ b/serialization/.jvm/src/main/scala/com/karasiq/shadowcloud/serialization/kryo/TimestampSerializer.scala @@ -8,7 +8,7 @@ import com.karasiq.shadowcloud.model.Timestamp private[kryo] final class TimestampSerializer extends chill.KSerializer[Timestamp](false, true) { def read(kryo: Kryo, input: Input, `type`: Class[Timestamp]): Timestamp = { - val created = input.readLong(true) + val created = input.readLong(true) val modifiedOffset = input.readLong(true) Timestamp(created, created + modifiedOffset) } @@ -18,4 +18,4 @@ private[kryo] final class TimestampSerializer extends chill.KSerializer[Timestam output.writeLong(timestamp.created, true) output.writeLong(modifiedOffset, true) } -} \ No newline at end of file +} diff --git a/serialization/.jvm/src/main/scala/com/karasiq/shadowcloud/serialization/kryo/WrappedConfigSerializer.scala b/serialization/.jvm/src/main/scala/com/karasiq/shadowcloud/serialization/kryo/WrappedConfigSerializer.scala index 701141a2..e04156a6 100644 --- a/serialization/.jvm/src/main/scala/com/karasiq/shadowcloud/serialization/kryo/WrappedConfigSerializer.scala +++ b/serialization/.jvm/src/main/scala/com/karasiq/shadowcloud/serialization/kryo/WrappedConfigSerializer.scala @@ -8,11 +8,10 @@ import com.typesafe.config.Config import com.karasiq.shadowcloud.config.{WrappedConfig, WrappedConfigFactory} import com.karasiq.shadowcloud.serialization.internal.CompanionReflectSerializer -private[kryo] final class WrappedConfigSerializer - extends chill.KSerializer[WrappedConfig](false, true) with CompanionReflectSerializer { +private[kryo] final class WrappedConfigSerializer extends chill.KSerializer[WrappedConfig](false, true) with CompanionReflectSerializer { def read(kryo: Kryo, input: Input, `type`: Class[WrappedConfig]): WrappedConfig = { - val config = kryo.readObject(input, classOf[Config]) + val config = kryo.readObject(input, classOf[Config]) val factory = getCompanion[WrappedConfigFactory[_ <: WrappedConfig]](`type`) factory.apply(config) } diff --git a/serialization/src/main/scala/com/karasiq/shadowcloud/serialization/json/SCJsonEncoders.scala b/serialization/src/main/scala/com/karasiq/shadowcloud/serialization/json/SCJsonEncoders.scala index dec06cd0..f911041b 100644 --- a/serialization/src/main/scala/com/karasiq/shadowcloud/serialization/json/SCJsonEncoders.scala +++ b/serialization/src/main/scala/com/karasiq/shadowcloud/serialization/json/SCJsonEncoders.scala @@ -34,12 +34,11 @@ trait SCJsonEncoders { implicit val symmetricEncryptionParametersFormat = Json.format[SymmetricEncryptionParameters] implicit val asymmetricEncryptionParametersFormat = Json.format[AsymmetricEncryptionParameters] implicit val encryptionParametersFormat = Format[EncryptionParameters]( - Reads( - value ⇒ - (value \ "_key_type").as[String] match { - case "symmetric" ⇒ symmetricEncryptionParametersFormat.reads(value) - case "asymmetric" ⇒ asymmetricEncryptionParametersFormat.reads(value) - } + Reads(value ⇒ + (value \ "_key_type").as[String] match { + case "symmetric" ⇒ symmetricEncryptionParametersFormat.reads(value) + case "asymmetric" ⇒ asymmetricEncryptionParametersFormat.reads(value) + } ), Writes { case s: SymmetricEncryptionParameters ⇒ symmetricEncryptionParametersFormat.writes(s) + ("_key_type" → JsString("symmetric")) diff --git a/server/api-routes/src/main/scala/com/karasiq/shadowcloud/server/http/SCAkkaHttpFileRoutes.scala b/server/api-routes/src/main/scala/com/karasiq/shadowcloud/server/http/SCAkkaHttpFileRoutes.scala index c6582b88..abfea182 100644 --- a/server/api-routes/src/main/scala/com/karasiq/shadowcloud/server/http/SCAkkaHttpFileRoutes.scala +++ b/server/api-routes/src/main/scala/com/karasiq/shadowcloud/server/http/SCAkkaHttpFileRoutes.scala @@ -34,17 +34,16 @@ trait SCAkkaHttpFileRoutes { self: SCAkkaHttpApiRoutes with SCHttpServerSettings SCFileDirectives.writeFile(regionId, path, dataStream) } ~ - (pathPrefix("upload_form") & extractMaterializer) { implicit m => + (pathPrefix("upload_form") & extractMaterializer) { implicit m ⇒ (path(Segment / SCPath) & entity(as[Multipart.FormData])) { (regionId, path, entity) ⇒ val fileEntity = entity.parts .filter(_.name == "file") - .map(p => (p.filename.getOrElse("default"), p.entity.withoutSizeLimit().dataBytes.mapMaterializedValue(_ => NotUsed))) + .map(p ⇒ (p.filename.getOrElse("default"), p.entity.withoutSizeLimit().dataBytes.mapMaterializedValue(_ ⇒ NotUsed))) .mapMaterializedValue(_ ⇒ NotUsed) .runWith(Sink.head) - onSuccess(fileEntity) { - case (name, bytes) => - SCFileDirectives.writeFile(regionId, path / name, bytes) + onSuccess(fileEntity) { case (name, bytes) ⇒ + SCFileDirectives.writeFile(regionId, path / name, bytes) } } } @@ -98,44 +97,44 @@ trait SCAkkaHttpFileRoutes { self: SCAkkaHttpApiRoutes with SCHttpServerSettings } def writeFiles(regionId: RegionId, path: Path, stream: Source[(String, Source[ByteString, NotUsed]), NotUsed]): Route = - extractMaterializer { implicit m => + extractMaterializer { implicit m ⇒ val resultStream = stream - .flatMapConcat { - case (name, bytes) => - bytes - .via(sc.streams.metadata.writeFileAndMetadata(regionId, path / name)) - .log("uploaded-files") - .map(_._1) + .flatMapConcat { case (name, bytes) ⇒ + bytes + .via(sc.streams.metadata.writeFileAndMetadata(regionId, path / name)) + .log("uploaded-files") + .map(_._1) } .fold(Vector.empty[File])(_ :+ _) .map(SCApiInternals.apiEncoding.encodeFiles(_)) .runWith(Sink.head) - onSuccess(resultStream) { result => + onSuccess(resultStream) { result ⇒ complete(StatusCodes.OK, HttpEntity(SCApiInternals.apiContentType, result)) } } - def writeFile(regionId: RegionId, path: Path, stream: Source[ByteString, NotUsed]): Route = extractMaterializer{ implicit m => + def writeFile(regionId: RegionId, path: Path, stream: Source[ByteString, NotUsed]): Route = extractMaterializer { implicit m ⇒ val resultStream = stream .via(sc.streams.metadata.writeFileAndMetadata(regionId, path)) .log("uploaded-files") .map(_._1) .runWith(Sink.head) - onSuccess(resultStream) { file => + onSuccess(resultStream) { file ⇒ complete(StatusCodes.OK, HttpEntity(SCApiInternals.apiContentType, SCApiInternals.apiEncoding.encodeFile(file))) } } def readChunkStream(regionId: RegionId, chunks: Seq[Chunk], fileName: String = ""): Route = { - val contentType = try { - ContentType(MediaTypes.forExtension(Utils.getFileExtensionLowerCase(fileName)), () ⇒ HttpCharsets.`UTF-8`) - } catch { - case NonFatal(_) ⇒ - ContentTypes.NoContentType - // ContentTypes.`application/octet-stream` - } + val contentType = + try { + ContentType(MediaTypes.forExtension(Utils.getFileExtensionLowerCase(fileName)), () ⇒ HttpCharsets.`UTF-8`) + } catch { + case NonFatal(_) ⇒ + ContentTypes.NoContentType + // ContentTypes.`application/octet-stream` + } def createEntity(dataStream: Source[ByteString, NotUsed], contentLength: Long) = { if (contentLength > 0) { diff --git a/server/api-routes/src/main/scala/com/karasiq/shadowcloud/server/http/SCWebSocketRoutes.scala b/server/api-routes/src/main/scala/com/karasiq/shadowcloud/server/http/SCWebSocketRoutes.scala index 4de8936b..e63b3413 100644 --- a/server/api-routes/src/main/scala/com/karasiq/shadowcloud/server/http/SCWebSocketRoutes.scala +++ b/server/api-routes/src/main/scala/com/karasiq/shadowcloud/server/http/SCWebSocketRoutes.scala @@ -28,13 +28,13 @@ trait SCWebSocketRoutes { self: SCAkkaHttpApiRoutes with SCHttpServerSettings wi Source .actorRef[LogEvent](PartialFunction.empty, PartialFunction.empty, 100, OverflowStrategy.dropBuffer) - .mapMaterializedValue { actor => + .mapMaterializedValue { actor ⇒ Seq(classOf[Info], classOf[Warning], classOf[Error]).foreach(as.eventStream.subscribe(actor, _)) NotUsed } .filterNot(_.level == LogLevels.Debug) .collect { - case e: LogEventWithCause if e.cause != null => + case e: LogEventWithCause if e.cause != null ⇒ val stack = { val sw = new StringWriter() val pw = new PrintWriter(sw) @@ -44,13 +44,13 @@ trait SCWebSocketRoutes { self: SCAkkaHttpApiRoutes with SCHttpServerSettings wi } s"${formatTime(e.timestamp)} (${e.logSource}) - ${e.message}\n$stack" - case e: LogEvent if !blacklist.exists(e.message.toString.contains) => + case e: LogEvent if !blacklist.exists(e.message.toString.contains) ⇒ s"${formatTime(e.timestamp)} (${e.logSource}) - ${e.message}" } } def scWebSocketRoutes: Route = path("log") { - extractActorSystem { implicit as => + extractActorSystem { implicit as ⇒ val flow = Flow.fromSinkAndSource(Sink.ignore, createLogStream).map(TextMessage(_)) handleWebSocketMessages(flow) } diff --git a/server/api-routes/src/main/scala/com/karasiq/shadowcloud/server/http/api/ShadowCloudApiImpl.scala b/server/api-routes/src/main/scala/com/karasiq/shadowcloud/server/http/api/ShadowCloudApiImpl.scala index c46d220a..652b9583 100644 --- a/server/api-routes/src/main/scala/com/karasiq/shadowcloud/server/http/api/ShadowCloudApiImpl.scala +++ b/server/api-routes/src/main/scala/com/karasiq/shadowcloud/server/http/api/ShadowCloudApiImpl.scala @@ -53,14 +53,12 @@ private[server] final class ShadowCloudApiImpl(sc: ShadowCloudExtension) extends } sc.ops.supervisor.getSnapshot().map { snapshot ⇒ - val storages = snapshot.storages.map { - case (storageId, storage) ⇒ - (storageId, toSerializableStorageStatus(storage)) + val storages = snapshot.storages.map { case (storageId, storage) ⇒ + (storageId, toSerializableStorageStatus(storage)) } - val regions = snapshot.regions.map { - case (regionId, region) ⇒ - (regionId, toSerializableRegionStatus(region)) + val regions = snapshot.regions.map { case (regionId, region) ⇒ + (regionId, toSerializableRegionStatus(region)) } RegionStateReport(regions, storages) diff --git a/server/api-routes/src/test/scala/com/karasiq/shadowcloud/server/http/test/SCHttpServerTest.scala b/server/api-routes/src/test/scala/com/karasiq/shadowcloud/server/http/test/SCHttpServerTest.scala index 3f703395..c36b7c56 100644 --- a/server/api-routes/src/test/scala/com/karasiq/shadowcloud/server/http/test/SCHttpServerTest.scala +++ b/server/api-routes/src/test/scala/com/karasiq/shadowcloud/server/http/test/SCHttpServerTest.scala @@ -30,19 +30,20 @@ class SCHttpServerTest extends FlatSpec with Matchers with ScalatestRouteTest wi implicit val routeTimeout = RouteTestTimeout(30 seconds) - val testRegionId = "testRegion" + val testRegionId = "testRegion" val testStorageId = "testStorage" - val route = Route.seal(TestServer.scRoutes) + val route = Route.seal(TestServer.scRoutes) val (testFileContent, testFile) = TestUtils.indexedBytes - val encodedPath = SCApiEncoding.toUrlSafe(apiEncoding.encodePath(testFile.path)) + val encodedPath = SCApiEncoding.toUrlSafe(apiEncoding.encodePath(testFile.path)) "Test server" should "upload file" in { val request = Post(s"/upload/$testRegionId/$encodedPath", HttpEntity(testFileContent)) .copy(headers = List(RawHeader("X-Requested-With", SCApiUtils.RequestedWith))) request ~> route ~> check { - val file = apiEncoding.decodeFile(entityAs[ByteString](implicitly[FromEntityUnmarshaller[ByteString]], implicitly[ClassTag[ByteString]], 30 seconds)) + val file = + apiEncoding.decodeFile(entityAs[ByteString](implicitly[FromEntityUnmarshaller[ByteString]], implicitly[ClassTag[ByteString]], 30 seconds)) file.path shouldBe testFile.path file.revision shouldBe 0 file.checksum.size shouldBe testFileContent.length @@ -64,7 +65,7 @@ class SCHttpServerTest extends FlatSpec with Matchers with ScalatestRouteTest wi request ~> route ~> check { val expectedContent = testFileContent.slice(0, 100) ++ - testFileContent.slice(100, 200) + testFileContent.slice(100, 200) entityAs[ByteString] shouldBe expectedContent } diff --git a/server/autowire-api/.js/src/main/scala/com/karasiq/shadowcloud/api/js/SCAjaxApiClient.scala b/server/autowire-api/.js/src/main/scala/com/karasiq/shadowcloud/api/js/SCAjaxApiClient.scala index 2b2410c0..c6d73f68 100644 --- a/server/autowire-api/.js/src/main/scala/com/karasiq/shadowcloud/api/js/SCAjaxApiClient.scala +++ b/server/autowire-api/.js/src/main/scala/com/karasiq/shadowcloud/api/js/SCAjaxApiClient.scala @@ -17,18 +17,16 @@ trait SCAjaxApiClient[Reader[_], Writer[_]] extends SCApiClient[ByteString, Read def doCall(req: Request): Future[ByteString] = { val payload = encodePayload(req.args) - val url = ("api" +: req.path).mkString("/", "/", "") + val url = ("api" +: req.path).mkString("/", "/", "") val headers = Map( - "SC-Accept" → payloadContentType, - "Content-Type" → payloadContentType, + "SC-Accept" → payloadContentType, + "Content-Type" → payloadContentType, "X-Requested-With" → SCApiUtils.RequestedWith ) - Ajax.post(url, payload.toByteBuffer, /* 15000 */ 0, headers, responseType = "arraybuffer") - .reportFailure - .responseBytes + Ajax.post(url, payload.toByteBuffer, /* 15000 */ 0, headers, responseType = "arraybuffer").reportFailure.responseBytes } } -object SCAjaxJsonApiClient extends SCAjaxApiClient[Reads, Writes] with SCJsonApiClient -object SCAjaxBooPickleApiClient extends SCAjaxApiClient[Pickler, Pickler] with SCBooPickleApiClient \ No newline at end of file +object SCAjaxJsonApiClient extends SCAjaxApiClient[Reads, Writes] with SCJsonApiClient +object SCAjaxBooPickleApiClient extends SCAjaxApiClient[Pickler, Pickler] with SCBooPickleApiClient diff --git a/server/autowire-api/.js/src/main/scala/com/karasiq/shadowcloud/api/js/utils/AjaxUtils.scala b/server/autowire-api/.js/src/main/scala/com/karasiq/shadowcloud/api/js/utils/AjaxUtils.scala index e70d49ef..9d32c642 100644 --- a/server/autowire-api/.js/src/main/scala/com/karasiq/shadowcloud/api/js/utils/AjaxUtils.scala +++ b/server/autowire-api/.js/src/main/scala/com/karasiq/shadowcloud/api/js/utils/AjaxUtils.scala @@ -8,7 +8,7 @@ import org.scalajs.dom.XMLHttpRequest object AjaxUtils { implicit val AjaxExecutionContext: ExecutionContext = scala.scalajs.concurrent.JSExecutionContext.Implicits.queue - val BytesResponseType = "arraybuffer" + val BytesResponseType = "arraybuffer" implicit class AjaxResultOps(private val future: Future[XMLHttpRequest]) extends AnyVal { def responseBytes: Future[ByteString] = { diff --git a/server/autowire-api/src/main/scala/com/karasiq/shadowcloud/api/SCApiServer.scala b/server/autowire-api/src/main/scala/com/karasiq/shadowcloud/api/SCApiServer.scala index 611f5d8a..be61de94 100644 --- a/server/autowire-api/src/main/scala/com/karasiq/shadowcloud/api/SCApiServer.scala +++ b/server/autowire-api/src/main/scala/com/karasiq/shadowcloud/api/SCApiServer.scala @@ -6,4 +6,4 @@ import akka.util.ByteString trait SCApiServer[Value, Reader[_], Writer[_]] extends autowire.Server[Value, Reader, Writer] with SCApiMeta { def decodePayload(payload: ByteString): Map[String, Value] -} \ No newline at end of file +} diff --git a/server/autowire-api/src/main/scala/com/karasiq/shadowcloud/api/SCApiUtils.scala b/server/autowire-api/src/main/scala/com/karasiq/shadowcloud/api/SCApiUtils.scala index 95278312..b43f8bf9 100644 --- a/server/autowire-api/src/main/scala/com/karasiq/shadowcloud/api/SCApiUtils.scala +++ b/server/autowire-api/src/main/scala/com/karasiq/shadowcloud/api/SCApiUtils.scala @@ -2,5 +2,5 @@ package com.karasiq.shadowcloud.api object SCApiUtils { val RequestedWith: String = "SCAjaxApiClient" - val PostHeaders = Map("X-Requested-With" → RequestedWith) + val PostHeaders = Map("X-Requested-With" → RequestedWith) } diff --git a/server/autowire-api/src/main/scala/com/karasiq/shadowcloud/api/ShadowCloudApi.scala b/server/autowire-api/src/main/scala/com/karasiq/shadowcloud/api/ShadowCloudApi.scala index 6b038e19..0dd2f188 100644 --- a/server/autowire-api/src/main/scala/com/karasiq/shadowcloud/api/ShadowCloudApi.scala +++ b/server/autowire-api/src/main/scala/com/karasiq/shadowcloud/api/ShadowCloudApi.scala @@ -47,7 +47,12 @@ trait ShadowCloudApi { // ----------------------------------------------------------------------- def getKeys(): Future[KeyChain] def modifyKey(keyId: KeyId, regionSet: Set[RegionId] = Set.empty, forEncryption: Boolean, forDecryption: Boolean): Future[Done] - def generateKey(regionSet: Set[RegionId] = Set.empty, forEncryption: Boolean = true, forDecryption: Boolean = true, props: SerializedProps = SerializedProps.empty): Future[KeySet] + def generateKey( + regionSet: Set[RegionId] = Set.empty, + forEncryption: Boolean = true, + forDecryption: Boolean = true, + props: SerializedProps = SerializedProps.empty + ): Future[KeySet] def addKey(key: KeySet, regionSet: Set[RegionId] = Set.empty, forEncryption: Boolean = true, forDecryption: Boolean = true): Future[KeySet] // ----------------------------------------------------------------------- diff --git a/server/autowire-api/src/main/scala/com/karasiq/shadowcloud/api/boopickle/SCBooPickleApiClient.scala b/server/autowire-api/src/main/scala/com/karasiq/shadowcloud/api/boopickle/SCBooPickleApiClient.scala index 512dea89..66180399 100644 --- a/server/autowire-api/src/main/scala/com/karasiq/shadowcloud/api/boopickle/SCBooPickleApiClient.scala +++ b/server/autowire-api/src/main/scala/com/karasiq/shadowcloud/api/boopickle/SCBooPickleApiClient.scala @@ -8,7 +8,7 @@ import com.karasiq.shadowcloud.api.SCApiClient trait SCBooPickleApiClient extends SCApiClient[ByteString, Pickler, Pickler] with SCBooPickleApi { import encoding.implicits._ - def encodePayload(value: Map[String, ByteString]) = encoding.encode(value) + def encodePayload(value: Map[String, ByteString]) = encoding.encode(value) def read[Result: boopickle.Pickler](p: ByteString) = encoding.decode[Result](p) - def write[Result: boopickle.Pickler](r: Result) = encoding.encode(r) + def write[Result: boopickle.Pickler](r: Result) = encoding.encode(r) } diff --git a/server/autowire-api/src/test/scala/com/shadowcloud/api/test/ApiEncodingTest.scala b/server/autowire-api/src/test/scala/com/shadowcloud/api/test/ApiEncodingTest.scala index b9455d38..94405868 100644 --- a/server/autowire-api/src/test/scala/com/shadowcloud/api/test/ApiEncodingTest.scala +++ b/server/autowire-api/src/test/scala/com/shadowcloud/api/test/ApiEncodingTest.scala @@ -13,7 +13,7 @@ class ApiEncodingTest extends FlatSpec with Matchers { "BooPickle" should "encode diff" in { val enc = SCBooPickleEncoding import enc.implicits._ - val bytes = enc.encode(testDiff) + val bytes = enc.encode(testDiff) val decoded = enc.decode[IndexDiff](bytes) decoded shouldBe testDiff } @@ -21,7 +21,7 @@ class ApiEncodingTest extends FlatSpec with Matchers { "JSON" should "encode diff" in { val enc = SCJsonEncoding import enc.implicits._ - val bytes = enc.encode(testDiff) + val bytes = enc.encode(testDiff) val decoded = enc.decode[IndexDiff](bytes) decoded shouldBe testDiff } diff --git a/server/src/main/scala/com/karasiq/shadowcloud/server/http/SCAkkaHttpServer.scala b/server/src/main/scala/com/karasiq/shadowcloud/server/http/SCAkkaHttpServer.scala index aef0b3e0..71902610 100644 --- a/server/src/main/scala/com/karasiq/shadowcloud/server/http/SCAkkaHttpServer.scala +++ b/server/src/main/scala/com/karasiq/shadowcloud/server/http/SCAkkaHttpServer.scala @@ -10,8 +10,6 @@ import com.karasiq.shadowcloud.server.http.static.SCAkkaHttpStaticRoutes import com.karasiq.shadowcloud.server.http.webzinc.SCAkkaHttpWebZincRoutes import org.apache.commons.codec.binary.Hex - - object SCAkkaHttpServer { def apply(sc: ShadowCloudExtension): SCAkkaHttpServer = { new SCAkkaHttpServer(sc) @@ -29,7 +27,7 @@ class SCAkkaHttpServer(protected val sc: ShadowCloudExtension) // Route // ----------------------------------------------------------------------- lazy val scWebAppRoutes: Route = httpServerConfig.passwordHash match { - case Some(passHash) => + case Some(passHash) ⇒ def hashString(str: String) = { val sha256 = MessageDigest.getInstance("SHA-256") val digestBytes = sha256.digest(str.getBytes) @@ -37,13 +35,14 @@ class SCAkkaHttpServer(protected val sc: ShadowCloudExtension) } authenticateBasic( - "shadowcloud", { - case c @ Credentials.Provided("sc") if c.verify(passHash.toLowerCase, hashString) => Some(allRoutes) - case _ => None + "shadowcloud", + { + case c @ Credentials.Provided("sc") if c.verify(passHash.toLowerCase, hashString) ⇒ Some(allRoutes) + case _ ⇒ None } )(identity) - case None => + case None ⇒ allRoutes } diff --git a/server/src/main/scala/com/karasiq/shadowcloud/server/http/SCAkkaHttpServerLifecycleHook.scala b/server/src/main/scala/com/karasiq/shadowcloud/server/http/SCAkkaHttpServerLifecycleHook.scala index 5c8c50dd..613f5887 100644 --- a/server/src/main/scala/com/karasiq/shadowcloud/server/http/SCAkkaHttpServerLifecycleHook.scala +++ b/server/src/main/scala/com/karasiq/shadowcloud/server/http/SCAkkaHttpServerLifecycleHook.scala @@ -6,7 +6,6 @@ import com.karasiq.shadowcloud.providers.LifecycleHook import scala.concurrent.Await - private[http] final class SCAkkaHttpServerLifecycleHook(sc: ShadowCloudExtension) extends LifecycleHook { @volatile private[this] var serverBinding: Http.ServerBinding = _ diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/SCFrontendMain.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/SCFrontendMain.scala index c04c0efd..316b9658 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/SCFrontendMain.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/SCFrontendMain.scala @@ -1,7 +1,5 @@ package com.karasiq.shadowcloud.webapp - - import com.karasiq.shadowcloud.webapp.components.SCFrontend import com.karasiq.shadowcloud.webapp.context.AppContext import org.scalajs.jquery._ diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/api/FileApi.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/api/FileApi.scala index 93eef2dd..f03baad9 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/api/FileApi.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/api/FileApi.scala @@ -15,27 +15,26 @@ trait FileApi { self: SCApiMeta ⇒ import com.karasiq.shadowcloud.api.js.utils.AjaxUtils._ def uploadFile(regionId: RegionId, path: Path, data: Ajax.InputData): (Rx[Int], Future[File]) = { - val (progress, future) = UploadUtils.uploadWithProgress(uploadFileUrl(regionId, path), data, - headers = SCApiUtils.PostHeaders, responseType = BytesResponseType) + val (progress, future) = + UploadUtils.uploadWithProgress(uploadFileUrl(regionId, path), data, headers = SCApiUtils.PostHeaders, responseType = BytesResponseType) (progress, future.map(encoding.decodeFile)) } def saveWebPage(regionId: RegionId, path: Path, url: String): Future[File] = { val apiUrl = URLPath(Path.root / "save_page" / regionId / SCApiEncoding.toUrlSafe(encoding.encodePath(path)), Map("url" → url)).toString - Ajax.post(apiUrl, headers = SCApiUtils.PostHeaders, responseType = BytesResponseType) + Ajax + .post(apiUrl, headers = SCApiUtils.PostHeaders, responseType = BytesResponseType) .responseBytes .map(encoding.decodeFile) } def downloadMostRecentFile(regionId: RegionId, path: Path, scope: IndexScope = IndexScope.default): Future[ByteString] = { - Ajax.get(mostRecentFileUrl(regionId, path, scope), responseType = BytesResponseType) - .responseBytes + Ajax.get(mostRecentFileUrl(regionId, path, scope), responseType = BytesResponseType).responseBytes } def downloadFile(regionId: RegionId, path: Path, id: FileId, scope: IndexScope = IndexScope.default): Future[ByteString] = { - Ajax.get(fileUrl(regionId, path, id, scope), responseType = BytesResponseType) - .responseBytes + Ajax.get(fileUrl(regionId, path, id, scope), responseType = BytesResponseType).responseBytes } def uploadFileUrl(regionId: RegionId, path: Path): String = { @@ -46,10 +45,11 @@ trait FileApi { self: SCApiMeta ⇒ require(!path.isRoot, "Not a file") val baseUrl = URLPath(_ / "download" / regionId / SCApiEncoding.toUrlSafe(encoding.encodePath(path)) / path.name) - val scopedUrl = if (scope == IndexScope.default) - baseUrl - else - baseUrl.withQuery("scope", SCApiEncoding.toUrlSafe(encoding.encodeScope(scope))) + val scopedUrl = + if (scope == IndexScope.default) + baseUrl + else + baseUrl.withQuery("scope", SCApiEncoding.toUrlSafe(encoding.encodeScope(scope))) scopedUrl.toString } @@ -59,11 +59,12 @@ trait FileApi { self: SCApiMeta ⇒ val baseUrl = URLPath(_ / "download" / regionId / SCApiEncoding.toUrlSafe(encoding.encodePath(path)) / path.name) .withQuery("file-id", fileId.toString) - val scopedUrl = if (scope == IndexScope.default) - baseUrl - else - baseUrl.withQuery("scope", SCApiEncoding.toUrlSafe(encoding.encodeScope(scope))) - + val scopedUrl = + if (scope == IndexScope.default) + baseUrl + else + baseUrl.withQuery("scope", SCApiEncoding.toUrlSafe(encoding.encodeScope(scope))) + scopedUrl.toString } } diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/SCContextBinding.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/SCContextBinding.scala index e9c29d31..cd8b18d3 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/SCContextBinding.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/SCContextBinding.scala @@ -41,7 +41,7 @@ private[webapp] class SCContextBinding()(implicit ctxOwner: Ctx.Owner) { def bindToString(value: Var[Option[String]]): Unit = { value.foreach { - case Some(encoded) if encoded.startsWith("/") => + case Some(encoded) if encoded.startsWith("/") ⇒ val Array(regionId, nodes @ _*) = encoded.tail.split("/") context() = EncodedContext(Some(regionId), Path(nodes)) @@ -49,11 +49,11 @@ private[webapp] class SCContextBinding()(implicit ctxOwner: Ctx.Owner) { val encodedContext = Json.parse(encoded).as[EncodedContext] context() = encodedContext - case Some("") => + case Some("") ⇒ context() = EncodedContext(None, Path.root) case _ ⇒ - // Ignore + // Ignore } context.triggerLater { diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/SCFrontend.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/SCFrontend.scala index b8ac3d7a..e00fcf68 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/SCFrontend.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/SCFrontend.scala @@ -22,7 +22,7 @@ object SCFrontend { class SCFrontend()(implicit val context: AppContext) { implicit val regionContext = RegionContext() - implicit val keysContext = KeysContext() + implicit val keysContext = KeysContext() val regionSwitcher = RegionSwitcher() @@ -42,8 +42,9 @@ class SCFrontend()(implicit val context: AppContext) { try { val themeCss = dom.document.head.querySelector("#sc-theme") themeSelector.linkModifier.applyTo(themeCss) - } catch { case NonFatal(_) ⇒ - themeSelector.applyTo(dom.document.head) + } catch { + case NonFatal(_) ⇒ + themeSelector.applyTo(dom.document.head) } // Frontend @@ -58,8 +59,11 @@ class SCFrontend()(implicit val context: AppContext) { def renderNavigationBar(): NavigationBar = { NavigationBar() - .withBrand(img(src := "/favicon.png", maxHeight := 100.pct, display.inline, - marginRight := 3.px), "shadowcloud", onclick := Callback.onClick(_ ⇒ themeSelector.nextTheme())) + .withBrand( + img(src := "/favicon.png", maxHeight := 100.pct, display.inline, marginRight := 3.px), + "shadowcloud", + onclick := Callback.onClick(_ ⇒ themeSelector.nextTheme()) + ) .withContentContainer(GridSystem.containerFluid(_, "sc-main-container".addClass)) .withStyles(NavigationBarStyle.default, NavigationBarStyle.staticTop) .withTabs( @@ -79,7 +83,7 @@ class SCFrontend()(implicit val context: AppContext) { case Some(folderContext) ⇒ regionSwitcher.scopeSelector.selectedScope.foreach(folderContext.scope.update) val folderController = FolderController.forFolderContext(folderContext) - val fileController = FileController.forFolderController(folderController) + val fileController = FileController.forFolderController(folderController) FoldersPanel()(context, folderContext, folderController, fileController) case None ⇒ @@ -92,4 +96,3 @@ class SCFrontend()(implicit val context: AppContext) { ) } } - diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/common/AppComponents.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/common/AppComponents.scala index 8c69155f..8c8efff5 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/common/AppComponents.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/common/AppComponents.scala @@ -50,19 +50,27 @@ object AppComponents { super.closeButton(onclick := Callback.onClick(_ ⇒ onClose())) }.renderTag(md: _*)) - def exportDialog(title: String, fileName: String, content: String, contentType: String = "application/json")( - implicit context: AppContext + def exportDialog(title: String, fileName: String, content: String, contentType: String = "application/json")(implicit + context: AppContext ): Modal = { def download(): Unit = Blobs.saveBlob(Blobs.fromString(content, contentType), fileName) Modal() .withTitle(title) - .withBody(FormInput.textArea("", content, rows := 30, readonly, onclick := { (e: MouseEvent) ⇒ - val textArea = e.target.asInstanceOf[TextArea] - textArea.focus() - textArea.select() - })) + .withBody( + FormInput.textArea( + "", + content, + rows := 30, + readonly, + onclick := { (e: MouseEvent) ⇒ + val textArea = e.target.asInstanceOf[TextArea] + textArea.focus() + textArea.select() + } + ) + ) .withButtons( Button(ButtonStyle.success)(context.locale.downloadFile, onclick := Callback.onClick(_ ⇒ download())), AppComponents.modalClose() @@ -75,28 +83,33 @@ object AppComponents { .withTitle(title) .withBody(Form(FormInput.textArea("", rows := 30, result.reactiveInput))) .withButtons( - AppComponents.modalSubmit(result.map(_.isEmpty).reactiveHide, onclick := Callback.onClick { _ ⇒ - submit(result.now) - result.kill() - }), + AppComponents.modalSubmit( + result.map(_.isEmpty).reactiveHide, + onclick := Callback.onClick { _ ⇒ + submit(result.now) + result.kill() + } + ), AppComponents.modalClose() ) } def disabledIf(rx: Rx[Boolean]): Modifier = (t: Element) ⇒ { - rx.foreach( - value ⇒ - if (value) t.setAttribute("disabled", "") - else t.removeAttribute("disabled") + rx.foreach(value ⇒ + if (value) t.setAttribute("disabled", "") + else t.removeAttribute("disabled") ) } def idSelect(title: String, ids: Iterable[String], selected: Iterable[String] = Nil): (FormSelect, Element) = { lazy val select: FormSelect = FormInput.multipleSelect( - a(title, onclick := Callback.onClick { _ ⇒ - if (select.selected.now.toSet == ids.toSet) select.selected() = Nil - else select.selected() = ids.toVector - }), + a( + title, + onclick := Callback.onClick { _ ⇒ + if (select.selected.now.toSet == ids.toSet) select.selected() = Nil + else select.selected() = ids.toVector + } + ), ids.map(id ⇒ FormSelectOption(id, id)).toVector ) select.selected() = selected.toVector diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/common/AppIcons.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/common/AppIcons.scala index 45f134e9..428d09cd 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/common/AppIcons.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/common/AppIcons.scala @@ -11,40 +11,40 @@ object AppIcons { val foldersView: Icon = "folder" val regionsView: Icon = "server" - val preview: Icon = "file-image-o" + val preview: Icon = "file-image-o" val metadata: Icon = "file-code-o" - val file: Icon = "file-o" - val folder: Icon = "folder-o" + val file: Icon = "file-o" + val folder: Icon = "folder-o" val folderOpen: Icon = "folder-open-o" - val refresh: Icon = "refresh" - val create: Icon = "plus" - val delete: Icon = "trash" - val rename: Icon = "pencil-square-o" + val refresh: Icon = "refresh" + val create: Icon = "plus" + val delete: Icon = "trash" + val rename: Icon = "pencil-square-o" val compress: Icon = "compress" - val play: Icon = "film" - val download: Icon = "download" - val upload: Icon = "upload" - val viewText: Icon = "file-text-o" - val editText: Icon = "edit" - val inspect: Icon = "user-secret" - val repair: Icon = "wrench" + val play: Icon = "film" + val download: Icon = "download" + val upload: Icon = "upload" + val viewText: Icon = "file-text-o" + val editText: Icon = "edit" + val inspect: Icon = "user-secret" + val repair: Icon = "wrench" val changeView: Icon = "list-alt" - val revisions: Icon = "clone" - val availability: Icon = "balance-scale" + val revisions: Icon = "clone" + val availability: Icon = "balance-scale" val partiallyAvailable: Icon = "exclamation-triangle" - val fullyAvailable: Icon = "check" + val fullyAvailable: Icon = "check" - val region: Icon = "microchip" + val region: Icon = "microchip" val currentScope: Icon = "check" val historyScope: Icon = "clock-o" val register: Icon = "plug" - val suspend: Icon = "pause" - val resume: Icon = "play" + val suspend: Icon = "pause" + val resume: Icon = "play" val logs: Icon = "book" diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/common/DateInput.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/common/DateInput.scala index 5f6b3a9a..3792dddd 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/common/DateInput.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/common/DateInput.scala @@ -17,8 +17,7 @@ object DateInput { } def toTimestamp(date: LocalDate): Long = { - date - .atStartOfDay + date.atStartOfDay .atZone(ZoneId.systemDefault) .toInstant .toEpochMilli @@ -83,4 +82,3 @@ private[components] class DatePickerPopup(val selectedDate: Var[Option[LocalDate selectedDate.foreach(writeDate(elem, _)) } } - diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/common/Dropzone.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/common/Dropzone.scala index af56a983..d6bbc8f1 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/common/Dropzone.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/common/Dropzone.scala @@ -19,15 +19,15 @@ class Dropzone(element: js.Any, options: js.Object) extends js.Object { } object Dropzone { - def apply(regionId: RegionId, path: () => Path, onSuccess: dom.File => Unit): Modifier = { element: dom.Element => + def apply(regionId: RegionId, path: () ⇒ Path, onSuccess: dom.File ⇒ Unit): Modifier = { element: dom.Element ⇒ val dz = new Dropzone( element, js.Dynamic .literal( - url = { _: js.Any => + url = { _: js.Any ⇒ URLPath(Path.root / "upload_form" / regionId / SCApiEncoding.toUrlSafe(AjaxApi.encoding.encodePath(path()))).toString }, - headers = js.Dynamic.literal("X-Requested-With" -> SCApiUtils.RequestedWith, "Accept" -> AjaxApi.payloadContentType), + headers = js.Dynamic.literal("X-Requested-With" → SCApiUtils.RequestedWith, "Accept" → AjaxApi.payloadContentType), maxFilesize = null, timeout = -1, parallelUploads = 4 diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/common/Pell.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/common/Pell.scala index db2d039d..d48142bc 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/common/Pell.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/common/Pell.scala @@ -24,7 +24,7 @@ class TurndownService(options: js.Dynamic) extends js.Object { object Pell { abstract class Editor(implicit ac: AppContext) extends BootstrapHtmlComponent { val submitting = Var(false) - val html = Var("") + val html = Var("") def onSubmit(): Unit @@ -35,31 +35,34 @@ object Pell { val options = js.Dynamic.literal( element = container, - onChange = (s: String) => html() = s + onChange = (s: String) ⇒ html() = s ) - dom.window.setTimeout(() => { - PellJS.init(options) - val editor = container.asInstanceOf[js.Dynamic].content.asInstanceOf[dom.Element] + dom.window.setTimeout( + () ⇒ { + PellJS.init(options) + val editor = container.asInstanceOf[js.Dynamic].content.asInstanceOf[dom.Element] - html.trigger { - if (editor.innerHTML != html.now) - editor.innerHTML = html.now - } - }, 0) + html.trigger { + if (editor.innerHTML != html.now) + editor.innerHTML = html.now + } + }, + 0 + ) div( parent, Button(ButtonStyle.success, block = true)( ac.locale.submit, - onclick := Callback.onClick(_ => if (!submitting.now) onSubmit()), + onclick := Callback.onClick(_ ⇒ if (!submitting.now) onSubmit()), "disabled".classIf(submitting) ) ) } } - def apply(f: Editor => Unit)(implicit ac: AppContext): Editor = new Editor { + def apply(f: Editor ⇒ Unit)(implicit ac: AppContext): Editor = new Editor { override def onSubmit(): Unit = f(this) } diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/common/TextEditor.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/common/TextEditor.scala index f9a0b1b6..772acfa9 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/common/TextEditor.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/common/TextEditor.scala @@ -16,13 +16,13 @@ object TextEditor { def memoized(key: String)(_onSubmit: TextEditor ⇒ Unit)(implicit context: AppContext): TextEditor = { new TextEditor { override val text: Var[String] = LocalStorage.memoize(key) - def onSubmit(): Unit = _onSubmit(this) + def onSubmit(): Unit = _onSubmit(this) } } } sealed abstract class TextEditor(implicit context: AppContext) extends BootstrapHtmlComponent { - val text = Var("") + val text = Var("") val submitting = Var(false) def onSubmit(): Unit diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/file/FileActions.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/file/FileActions.scala index c497b806..a805ada6 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/file/FileActions.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/file/FileActions.scala @@ -17,16 +17,17 @@ import com.karasiq.shadowcloud.webapp.controllers.FileController import com.karasiq.shadowcloud.webapp.utils.ExportUtils object FileActions { - def apply(file: File, useId: Boolean = false)(implicit context: AppContext, - folderContext: FolderContext, - fileController: FileController): FileActions = { + def apply(file: File, useId: Boolean = false)(implicit + context: AppContext, + folderContext: FolderContext, + fileController: FileController + ): FileActions = { new FileActions(file, useId) } } -final class FileActions(file: File, useId: Boolean)(implicit context: AppContext, - folderContext: FolderContext, - fileController: FileController) extends BootstrapHtmlComponent { +final class FileActions(file: File, useId: Boolean)(implicit context: AppContext, folderContext: FolderContext, fileController: FileController) + extends BootstrapHtmlComponent { private[this] val deleted = Var(false) @@ -46,14 +47,18 @@ final class FileActions(file: File, useId: Boolean)(implicit context: AppContext } private[this] def renderEditor(): TagT = { - renderAction(context.locale.viewTextFile, AppIcons.viewText, onclick := Callback.onClick { _ ⇒ - Modal() - .withTitle(context.locale.viewTextFile) - .withBody(TextFileView(file)) - .withButtons(AppComponents.modalClose()) - .withDialogStyle(ModalDialogSize.large) - .show(backdrop = false) - }) + renderAction( + context.locale.viewTextFile, + AppIcons.viewText, + onclick := Callback.onClick { _ ⇒ + Modal() + .withTitle(context.locale.viewTextFile) + .withBody(TextFileView(file)) + .withButtons(AppComponents.modalClose()) + .withDialogStyle(ModalDialogSize.large) + .show(backdrop = false) + } + ) } private[this] def renderPlayer(): TagT = { @@ -128,7 +133,7 @@ final class FileActions(file: File, useId: Boolean)(implicit context: AppContext def undeleteFile(): Unit = { val createdFiles = deletedFiles.toVector - val result = Future.sequence(createdFiles.map(context.api.createFile(folderContext.regionId, _))) + val result = Future.sequence(createdFiles.map(context.api.createFile(folderContext.regionId, _))) result.foreach { _ ⇒ deleted() = false createdFiles.foreach(fileController.addFile) @@ -148,7 +153,7 @@ final class FileActions(file: File, useId: Boolean)(implicit context: AppContext } else { context.api.getFiles(folderContext.regionId, file.path, dropChunks = false, folderContext.scope.now).map(FileVersions.mostRecent) } */ - val jsonRx = fileFuture.map(ExportUtils.encodeFile).toRx("") + val jsonRx = fileFuture.map(ExportUtils.encodeFile).toRx("") Modal(context.locale.inspectFile, pre(jsonRx), AppComponents.modalClose(), dialogStyle = ModalDialogSize.large).show() } @@ -156,7 +161,6 @@ final class FileActions(file: File, useId: Boolean)(implicit context: AppContext } private[this] def renderAction(title: ModifierT, icon: IconModifier, linkMd: ModifierT*): TagT = { - div(AppComponents.iconLink(title, icon, linkMd:_*)) + div(AppComponents.iconLink(title, icon, linkMd: _*)) } } - diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/file/FileAvailabilityView.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/file/FileAvailabilityView.scala index 693f9389..ab291c7b 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/file/FileAvailabilityView.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/file/FileAvailabilityView.scala @@ -26,7 +26,7 @@ class FileAvailabilityView(file: File)(implicit context: AppContext, folderConte def renderTag(md: ModifierT*): TagT = { div( div(renderRepairLink()), - Rx(renderContent(availabilityRx.toRx(), md:_*)) + Rx(renderContent(availabilityRx.toRx(), md: _*)) ) } @@ -55,7 +55,7 @@ class FileAvailabilityView(file: File)(implicit context: AppContext, folderConte div(b(context.locale.total, ": ", f"${fileAvailability.totalPercentage}%.2f%%")), for ((storageId, percentage) ← sortedPercentages) yield div(icon(percentage), storageId, f" ($percentage%.2f%%)", textStyle(percentage)) - )(md:_*) + )(md: _*) } def renderRepairLink(): TagT = { @@ -74,7 +74,7 @@ class FileAvailabilityView(file: File)(implicit context: AppContext, folderConte } context.api.getRegion(folderContext.regionId).foreach { status ⇒ - val storagesSelector = FormInput.simpleMultipleSelect(context.locale.storages, status.storages.toSeq:_*) + val storagesSelector = FormInput.simpleMultipleSelect(context.locale.storages, status.storages.toSeq: _*) Modal(context.locale.repairFile) .withBody(Form(storagesSelector)) .withButtons( @@ -84,7 +84,11 @@ class FileAvailabilityView(file: File)(implicit context: AppContext, folderConte .show() } } - AppComponents.iconLink(context.locale.repairFile, AppIcons.repair, Bootstrap.textStyle.muted.className.classIf(repairing), onclick := Callback.onClick(_ ⇒ if (!repairing.now) showDialog())) + AppComponents.iconLink( + context.locale.repairFile, + AppIcons.repair, + Bootstrap.textStyle.muted.className.classIf(repairing), + onclick := Callback.onClick(_ ⇒ if (!repairing.now) showDialog()) + ) } } - diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/file/FileDownloadLink.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/file/FileDownloadLink.scala index 0baad2cd..083d77c1 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/file/FileDownloadLink.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/file/FileDownloadLink.scala @@ -1,7 +1,5 @@ package com.karasiq.shadowcloud.webapp.components.file - - import com.karasiq.bootstrap.Bootstrap.default._ import com.karasiq.shadowcloud.model.File import com.karasiq.shadowcloud.webapp.context.{AppContext, FolderContext} @@ -10,14 +8,13 @@ import rx.Rx import scalaTags.all._ object FileDownloadLink { - def apply(file: File, useId: Boolean = false)(title: Modifier*) - (implicit context: AppContext, folderContext: FolderContext): FileDownloadLink = { + def apply(file: File, useId: Boolean = false)(title: Modifier*)(implicit context: AppContext, folderContext: FolderContext): FileDownloadLink = { new FileDownloadLink(file, useId)(title) } } -final class FileDownloadLink(file: File, useId: Boolean)(title: Modifier*) - (implicit context: AppContext, folderContext: FolderContext) extends BootstrapHtmlComponent { +final class FileDownloadLink(file: File, useId: Boolean)(title: Modifier*)(implicit context: AppContext, folderContext: FolderContext) + extends BootstrapHtmlComponent { def renderTag(md: ModifierT*): TagT = { val downloadUrl = RxUtils.getDownloadLinkRx(file, useId) diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/file/FilePreview.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/file/FilePreview.scala index f11d39ff..cd9e4932 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/file/FilePreview.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/file/FilePreview.scala @@ -14,9 +14,7 @@ import scala.concurrent.Future import scala.scalajs.concurrent.JSExecutionContext.Implicits.queue object FilePreview { - case class PreviewVariants(image: Option[Metadata.Thumbnail] = None, - text: Option[Metadata.Text] = None, - files: Option[Metadata.FileList] = None) + case class PreviewVariants(image: Option[Metadata.Thumbnail] = None, text: Option[Metadata.Text] = None, files: Option[Metadata.FileList] = None) object PreviewVariants { val empty = PreviewVariants() @@ -46,18 +44,17 @@ class FilePreview(regionId: RegionId, file: File)(implicit context: AppContext) def renderTag(md: ModifierT*): TagT = { val image = Rx(previewsRx().image.map(MetadataView.thumbnail)) - val text = Rx(previewsRx().text.map(MetadataView.text)) + val text = Rx(previewsRx().text.map(MetadataView.text)) val files = Rx(previewsRx().files.map(MetadataView.fileList)) div( - for (preview ← Seq( - image, - text, - files - )) yield preview.map(_.fold((): Frag)(GridSystem.mkRow(_))) - )(md:_*) + for ( + preview ← Seq( + image, + text, + files + ) + ) yield preview.map(_.fold((): Frag)(GridSystem.mkRow(_))) + )(md: _*) } } - - - diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/file/FileRevisionsView.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/file/FileRevisionsView.scala index b4d03640..5e97e7ff 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/file/FileRevisionsView.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/file/FileRevisionsView.scala @@ -21,8 +21,8 @@ object FileRevisionsView { } } -class FileRevisionsView(path: Path)(implicit context: AppContext, folderContext: FolderContext, - _fileController: FileController) extends BootstrapHtmlComponent { +class FileRevisionsView(path: Path)(implicit context: AppContext, folderContext: FolderContext, _fileController: FileController) + extends BootstrapHtmlComponent { lazy val filesRx = FileRevisionsView.getFilesRx(path) @@ -51,7 +51,6 @@ class FileRevisionsView(path: Path)(implicit context: AppContext, folderContext: } def renderTag(md: ModifierT*): TagT = { - fileList.renderTag(md:_*) + fileList.renderTag(md: _*) } } - diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/file/FileView.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/file/FileView.scala index 11ea9a9b..1a385868 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/file/FileView.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/file/FileView.scala @@ -39,7 +39,9 @@ class FileView(file: File, useId: Boolean)(implicit context: AppContext, folderC val preview = Some(NavigationTab(context.locale.preview, "preview", AppIcons.preview, FilePreview(folderContext.regionId, file))) .filter(_ ⇒ isPreviewAvailable) - val metadata = Some(NavigationTab(context.locale.metadata, "metadata", AppIcons.metadata, MetadataListView(folderContext.regionId, file, metadatasAvailable))) + val metadata = Some( + NavigationTab(context.locale.metadata, "metadata", AppIcons.metadata, MetadataListView(folderContext.regionId, file, metadatasAvailable)) + ) .filter(_ ⇒ metadatasAvailable().nonEmpty) preview.toSeq ++ metadata ++ Seq( diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/file/MediaFileView.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/file/MediaFileView.scala index f3c59a74..fe05fbfb 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/file/MediaFileView.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/file/MediaFileView.scala @@ -84,11 +84,10 @@ class MediaFileView(file: File, useId: Boolean)(implicit context: AppContext, fo renderAudioPlayer() else if (MediaFileView.isImageFile(file)) renderImage() - else + else renderVideoPlayer() } div(renderPlayer()) } } - diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/file/TextFileView.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/file/TextFileView.scala index 2deb77cb..0bcacc1a 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/file/TextFileView.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/file/TextFileView.scala @@ -20,14 +20,50 @@ object TextFileView { private[this] val TextFormats = Set("txt", "ini", "csv", "log") private[this] val CodeFormats = Set( - "sh", "clojure", "coffee", "c", "h", "cpp", "hpp", "cs", "d", "pas", "erl", - "fs", "go", "groovy", "hs", "java", "js", "json", "lua", "lisp", "md", "m", - "pl", "php", "py", "rb", "rust", "scala", "ss", "sql", "swift", "ts", "css", - "xml", "html", "xhtml", "conf" + "sh", + "clojure", + "coffee", + "c", + "h", + "cpp", + "hpp", + "cs", + "d", + "pas", + "erl", + "fs", + "go", + "groovy", + "hs", + "java", + "js", + "json", + "lua", + "lisp", + "md", + "m", + "pl", + "php", + "py", + "rb", + "rust", + "scala", + "ss", + "sql", + "swift", + "ts", + "css", + "xml", + "html", + "xhtml", + "conf" ) private[this] val RenderableFormats = Set( - "html", "htm", "xhtml", "md" + "html", + "htm", + "xhtml", + "md" ) def apply(file: File)(implicit context: AppContext, folderContext: FolderContext, fileController: FileController): TextFileView = { @@ -55,17 +91,17 @@ object TextFileView { } } -class TextFileView(_file: File)(implicit context: AppContext, folderContext: FolderContext, - fileController: FileController) extends BootstrapHtmlComponent { +class TextFileView(_file: File)(implicit context: AppContext, folderContext: FolderContext, fileController: FileController) + extends BootstrapHtmlComponent { val editorOpened = Var(false) - val fileRx = Var(_file) - val contentRx = Var("") + val fileRx = Var(_file) + val contentRx = Var("") fileRx.trigger(fetchFileContent()) def renderTag(md: ModifierT*): TagT = { val field = Rx { - val file = fileRx() + val file = fileRx() val content = contentRx() if (editorOpened()) { @@ -85,22 +121,27 @@ class TextFileView(_file: File)(implicit context: AppContext, folderContext: Fol } div( - AppComponents.iconLink(context.locale.edit, AppIcons.editText, onclick := Callback.onClick { _ ⇒ - editorOpened() = !editorOpened.now - }), + AppComponents.iconLink( + context.locale.edit, + AppIcons.editText, + onclick := Callback.onClick { _ ⇒ + editorOpened() = !editorOpened.now + } + ), div(field) ) } private[this] def fetchFileContent(): Unit = { - context.api.downloadFile(folderContext.regionId, fileRx.now.path, fileRx.now.id, folderContext.scope.now) + context.api + .downloadFile(folderContext.regionId, fileRx.now.path, fileRx.now.id, folderContext.scope.now) .map(_.utf8String) .foreach(contentRx.update) } - private[this] def editorOnSubmit[T](getText: T => String, submitting: T => Var[Boolean])(editor: T): Unit = { + private[this] def editorOnSubmit[T](getText: T ⇒ String, submitting: T ⇒ Var[Boolean])(editor: T): Unit = { submitting(editor)() = true - val oldFile = fileRx.now + val oldFile = fileRx.now val (_, future) = context.api.uploadFile(folderContext.regionId, oldFile.path, getText(editor)) future.onComplete(_ ⇒ submitting(editor)() = false) future.foreach { newFile ⇒ @@ -147,7 +188,7 @@ class TextFileView(_file: File)(implicit context: AppContext, folderContext: Fol lang.fold(HighlightJS.highlightAuto(source))(HighlightJS.highlight(_, source)).value }, renderer = MarkedRenderer(table = { (header: String, body: String) ⇒ - import scalatags.Text.all.{body => _, header => _, _} + import scalatags.Text.all.{body ⇒ _, header ⇒ _, _} table(`class` := "table table-striped", thead(raw(header)), tbody(raw(body))).render }), breaks = true, @@ -158,4 +199,3 @@ class TextFileView(_file: File)(implicit context: AppContext, folderContext: Fol div(raw(Marked(content, options))) } } - diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/folder/DragAndDrop.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/folder/DragAndDrop.scala index 8cce3c4b..8d3680ec 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/folder/DragAndDrop.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/folder/DragAndDrop.scala @@ -99,11 +99,11 @@ private[folder] object DragAndDrop { } private[this] object Attributes { - val path = "path" - val file = "file" - val folder = "folder" + val path = "path" + val file = "file" + val folder = "folder" val entityType = "entityType" - val regionId = "regionId" - val scope = "scope" + val regionId = "regionId" + val scope = "scope" } } diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/folder/FolderActions.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/folder/FolderActions.scala index c81aefb3..9ff46af1 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/folder/FolderActions.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/folder/FolderActions.scala @@ -12,50 +12,66 @@ import AppContext.JsExecutionContext import com.karasiq.shadowcloud.webapp.controllers.FolderController private[folder] object FolderActions { - def apply(regionId: RegionId, path: Path) - (implicit appContext: AppContext, folderContext: FolderContext, folderController: FolderController): FolderActions = { + def apply(regionId: RegionId, path: Path)(implicit + appContext: AppContext, + folderContext: FolderContext, + folderController: FolderController + ): FolderActions = { new FolderActions(regionId, path) } } -private[folder] class FolderActions(regionId: RegionId, path: Path) - (implicit context: AppContext, folderContext: FolderContext, folderController: FolderController) extends BootstrapHtmlComponent { +private[folder] class FolderActions(regionId: RegionId, path: Path)(implicit + context: AppContext, + folderContext: FolderContext, + folderController: FolderController +) extends BootstrapHtmlComponent { def renderTag(md: ModifierT*): TagT = { def showCreateFolderModal(): Unit = { - val folderNameRx = Var("") + val folderNameRx = Var("") val isFolderNameValid = folderNameRx.map(_.nonEmpty) Modal() .withTitle(context.locale.createFolder) .withBody(Form(FormInput.text(context.locale.name, folderNameRx.reactiveInputRead))) .withButtons( - Modal.button(context.locale.submit, Modal.dismiss, isFolderNameValid.reactiveShow, onclick := Callback.onClick { _ ⇒ - context.api.createFolder(regionId, path / folderNameRx.now).foreach(folderController.addFolder) - }), + Modal.button( + context.locale.submit, + Modal.dismiss, + isFolderNameValid.reactiveShow, + onclick := Callback.onClick { _ ⇒ + context.api.createFolder(regionId, path / folderNameRx.now).foreach(folderController.addFolder) + } + ), AppComponents.modalClose() ) .show() } def showRenameFolderModal(): Unit = { - val folderNameRx = Var(path.name) + val folderNameRx = Var(path.name) val isFolderNameValid = folderNameRx.map(_.nonEmpty) Modal() .withTitle(context.locale.rename) .withBody(Form(FormInput.text(context.locale.name, folderNameRx.reactiveInput))) .withButtons( - Modal.button(context.locale.submit, Modal.dismiss, isFolderNameValid.reactiveShow, onclick := Callback.onClick { _ ⇒ - for { - newFolder ← context.api.copyFolder(regionId, path, path.withName(folderNameRx.now), folderContext.scope.now) - oldFolder ← context.api.deleteFolder(regionId, path) - } { - folderController.deleteFolder(oldFolder) - folderController.addFolder(newFolder) + Modal.button( + context.locale.submit, + Modal.dismiss, + isFolderNameValid.reactiveShow, + onclick := Callback.onClick { _ ⇒ + for { + newFolder ← context.api.copyFolder(regionId, path, path.withName(folderNameRx.now), folderContext.scope.now) + oldFolder ← context.api.deleteFolder(regionId, path) + } { + folderController.deleteFolder(oldFolder) + folderController.addFolder(newFolder) + } } - }), + ), AppComponents.modalClose() ) .show() @@ -87,4 +103,3 @@ private[folder] class FolderActions(regionId: RegionId, path: Path) ) } } - diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/folder/FolderFileList.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/folder/FolderFileList.scala index ecdff1bd..eb06d903 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/folder/FolderFileList.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/folder/FolderFileList.scala @@ -21,8 +21,8 @@ object FolderFileList { } } -class FolderFileList(filesRx: Rx[Set[File]], flat: Boolean)( - implicit context: AppContext, +class FolderFileList(filesRx: Rx[Set[File]], flat: Boolean)(implicit + context: AppContext, folderContext: FolderContext, _fileController: FileController ) extends BootstrapHtmlComponent { diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/folder/FolderTree.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/folder/FolderTree.scala index 3952527a..22881969 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/folder/FolderTree.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/folder/FolderTree.scala @@ -100,7 +100,7 @@ class FolderTree(val path: Path)(implicit context: AppContext, folderContext: Fo private[this] def copyFile(file: File, readScope: IndexScope): Unit = { val newFileSet = context.api.copyFile(regionId, file, file.path.withParent(path), readScope) - newFileSet.foreach{ _ ⇒ + newFileSet.foreach { _ ⇒ folderController.update(path) Toastr.success(s"${file.path} successfully copied to $path") } @@ -181,10 +181,16 @@ class FolderTree(val path: Path)(implicit context: AppContext, folderContext: Fo span( a(href := "#", icon, styles, marginRight := 2.px, onclick := Callback.onClick(_ ⇒ opened() = !opened.now)), - a(href := "#", FolderTree.toPathString(path), styles, dragAndDropHandlers, onclick := Callback.onClick { _ ⇒ - folderContext.selected() = path - opened() = true - }), + a( + href := "#", + FolderTree.toPathString(path), + styles, + dragAndDropHandlers, + onclick := Callback.onClick { _ ⇒ + folderContext.selected() = path + opened() = true + } + ), actions ) } diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/folder/FoldersPanel.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/folder/FoldersPanel.scala index 2ca60887..7c7fc6a7 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/folder/FoldersPanel.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/folder/FoldersPanel.scala @@ -10,28 +10,28 @@ import com.karasiq.shadowcloud.webapp.controllers.{FileController, FolderControl import com.karasiq.shadowcloud.webapp.utils.RxUtils object FoldersPanel { - def apply()(implicit appContext: AppContext, - folderContext: FolderContext, - folderController: FolderController, - fileController: FileController): FoldersPanel = { + def apply()(implicit + appContext: AppContext, + folderContext: FolderContext, + folderController: FolderController, + fileController: FileController + ): FoldersPanel = { new FoldersPanel } } -class FoldersPanel(implicit context: AppContext, - folderContext: FolderContext, - folderController: FolderController, - fileController: FileController) extends BootstrapHtmlComponent { +class FoldersPanel(implicit context: AppContext, folderContext: FolderContext, folderController: FolderController, fileController: FileController) + extends BootstrapHtmlComponent { lazy val selectedFolder = RxUtils.getSelectedFolderRx - lazy val folderTree = FolderTree(Path.root) + lazy val folderTree = FolderTree(Path.root) lazy val folderFiles = FolderFileList(selectedFolder.map(_.files)) def renderTag(md: ModifierT*): TagT = { val currentFileView = folderFiles.selectedFile.map[Frag] { case Some(file) ⇒ FileView(file)(context, folderContext, folderFiles.fileController) - case None ⇒ () + case None ⇒ () } GridSystem.row( @@ -42,4 +42,3 @@ class FoldersPanel(implicit context: AppContext, ) } } - diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/folder/PreviewsFileList.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/folder/PreviewsFileList.scala index 3ac2a881..3f9d1433 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/folder/PreviewsFileList.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/folder/PreviewsFileList.scala @@ -9,14 +9,13 @@ import com.karasiq.shadowcloud.model.File import com.karasiq.shadowcloud.webapp.context.{AppContext, FolderContext} private[folder] object PreviewsFileList { - def apply(files: Rx[Seq[File]], selectedFile: Var[Option[File]]) - (implicit context: AppContext, fc: FolderContext): PreviewsFileList = { + def apply(files: Rx[Seq[File]], selectedFile: Var[Option[File]])(implicit context: AppContext, fc: FolderContext): PreviewsFileList = { new PreviewsFileList(files, selectedFile) } } -private[folder] class PreviewsFileList(files: Rx[Seq[File]], selectedFile: Var[Option[File]]) - (implicit context: AppContext, fc: FolderContext) extends BootstrapHtmlComponent { +private[folder] class PreviewsFileList(files: Rx[Seq[File]], selectedFile: Var[Option[File]])(implicit context: AppContext, fc: FolderContext) + extends BootstrapHtmlComponent { def renderTag(md: ModifierT*): TagT = { val filterField = Form(FormInput.text("", sorting.filter.reactiveInput)) @@ -35,15 +34,15 @@ private[folder] class PreviewsFileList(files: Rx[Seq[File]], selectedFile: Var[O div(textAlign.center, pagination.pageSelector, pagination.pages.map(_ <= 1).reactiveHide), div(sortLine, sorting.sortedFiles.map(_.length <= 1).reactiveHide), // hr(files.map(_.isEmpty).reactiveHide), - Rx(GridSystem.containerFluid(pagination.currentPageFiles().map(PreviewsFileListItem(_, selectedFile)):_*)) + Rx(GridSystem.containerFluid(pagination.currentPageFiles().map(PreviewsFileListItem(_, selectedFile)): _*)) ) } private[this] object sorting { val sortTypes = Seq(context.locale.name, context.locale.size, context.locale.modifiedDate) - val sortType = Var(0) - val sortDesc = Var(false) - val filter = Var("") + val sortType = Var(0) + val sortDesc = Var(false) + val filter = Var("") lazy val sortedFiles = Rx { val allFiles = files() @@ -53,10 +52,10 @@ private[folder] class PreviewsFileList(files: Rx[Seq[File]], selectedFile: Var[O else allFiles.filter(_.path.name.toLowerCase.contains(filterStr.toLowerCase)) } val sorted = sortTypes(sortType()) match { - case s if s == context.locale.name ⇒ filtered.sortBy(_.path.name) - case s if s == context.locale.size ⇒ filtered.sortBy(_.checksum.size) + case s if s == context.locale.name ⇒ filtered.sortBy(_.path.name) + case s if s == context.locale.size ⇒ filtered.sortBy(_.checksum.size) case s if s == context.locale.modifiedDate ⇒ filtered.sortBy(_.timestamp.lastModified) - case _ ⇒ filtered + case _ ⇒ filtered } if (sortDesc()) sorted.reverse else sorted } @@ -73,7 +72,7 @@ private[folder] class PreviewsFileList(files: Rx[Seq[File]], selectedFile: Var[O private[this] object pagination { val rowsPerPage = 10 - lazy val pages = sorting.sortedFiles.map(fs ⇒ (fs.length.toDouble / rowsPerPage).ceil.toInt) + lazy val pages = sorting.sortedFiles.map(fs ⇒ (fs.length.toDouble / rowsPerPage).ceil.toInt) lazy val pageSelector = PageSelector(pages) @@ -83,4 +82,3 @@ private[folder] class PreviewsFileList(files: Rx[Seq[File]], selectedFile: Var[O } } } - diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/folder/UploadForm.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/folder/UploadForm.scala index 9e95fa80..e3a4fd40 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/folder/UploadForm.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/folder/UploadForm.scala @@ -21,7 +21,7 @@ object UploadForm { private def isUrlList(text: String): Boolean = text.nonEmpty && text.lines.forall(_.matches("https?://[^\\s]+")) - private def newNoteName(text: String): String = { + private def newNoteName(text: String): String = { val firstLine = text.lines .map(_.trim) .find(_.nonEmpty) @@ -37,7 +37,7 @@ object UploadForm { private def uploadNoteOrPage(regionId: RegionId, path: Path, text: String, name: String)(implicit appContext: AppContext): Future[Seq[File]] = { if (isUrlList(text)) { Future - .sequence(text.lines.map(appContext.api.saveWebPage(regionId, path, _).map(Some(_)).recover { case _ => None }).toSeq) + .sequence(text.lines.map(appContext.api.saveWebPage(regionId, path, _).map(Some(_)).recover { case _ ⇒ None }).toSeq) .map(_.flatten) } else { val fileName = @@ -61,9 +61,13 @@ class UploadForm(implicit appContext: AppContext, folderContext: FolderContext, Form( action := "/", `class` := "dropzone", - Dropzone(folderContext.regionId, () => path, { file => - folderContext.update(path) - }) + Dropzone( + folderContext.regionId, + () ⇒ path, + { file ⇒ + folderContext.update(path) + } + ) ).render ) } @@ -101,11 +105,11 @@ class UploadForm(implicit appContext: AppContext, folderContext: FolderContext, FormInput.radio("Markdown", "markdown", "markdown", initialState = true) ) Form( - Pell { editor => + Pell { editor ⇒ editor.submitting() = true val extension = formatSelector.value.now match { - case "html" => "html" - case "markdown" => "md" + case "html" ⇒ "html" + case "markdown" ⇒ "md" } val resultFileName = @@ -118,13 +122,13 @@ class UploadForm(implicit appContext: AppContext, folderContext: FolderContext, else editor.html.now val future = appContext.api.uploadFile(folderContext.regionId, folderContext.selected.now / resultFileName, result)._2 - future.foreach { file => + future.foreach { file ⇒ Toastr.success(s"${file.path.name} successfully uploaded to ${file.path.parent}") fileController.addFile(file) editor.html() = "" fileName() = "" } - future.onComplete(_ => editor.submitting() = false) + future.onComplete(_ ⇒ editor.submitting() = false) }, formatSelector, FormInput.text("", fileName.reactiveInput, placeholder := "test.html") @@ -150,8 +154,13 @@ class UploadForm(implicit appContext: AppContext, folderContext: FolderContext, } def renderButton(md: ModifierT*): TagT = { - Button(ButtonStyle.info)(AppIcons.upload, appContext.locale.uploadFiles, md, onclick := Callback.onClick { _ ⇒ - renderModal().show() - }) + Button(ButtonStyle.info)( + AppIcons.upload, + appContext.locale.uploadFiles, + md, + onclick := Callback.onClick { _ ⇒ + renderModal().show() + } + ) } } diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/keys/KeysContext.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/keys/KeysContext.scala index 02e0eb95..bce5cce6 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/keys/KeysContext.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/keys/KeysContext.scala @@ -15,7 +15,7 @@ trait KeysContext { object KeysContext { def apply()(implicit appContext: AppContext, ctxOwner: Ctx.Owner): KeysContext = new KeysContext { private[this] val _keys = RxWithUpdate(KeyChain.empty)(_ ⇒ appContext.api.getKeys()) - def keys: Rx[KeyChain] = _keys.toRx - def updateAll(): Unit = _keys.update() + def keys: Rx[KeyChain] = _keys.toRx + def updateAll(): Unit = _keys.update() } } diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/keys/KeysView.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/keys/KeysView.scala index c4e69f57..7de9f4b3 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/keys/KeysView.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/keys/KeysView.scala @@ -31,16 +31,22 @@ class KeysView()(implicit context: AppContext, kc: KeysContext, rc: RegionContex val (activeKeys, disabledKeys) = keyChain.keys.partition(kp ⇒ kp.forEncryption || kp.forDecryption) val activeRows = activeKeys.map { kp ⇒ - val key = kp.key + val key = kp.key val style = if (kp.regionSet.isEmpty) TableRowStyle.active else TableRowStyle.info - TableRow(Seq(key.id.toString, key.encryption.method.algorithm, key.signing.method.algorithm), style, - onclick := Callback.onClick(_ ⇒ showKeyDialog(key, kp.regionSet, kp.forEncryption, kp.forDecryption))) + TableRow( + Seq(key.id.toString, key.encryption.method.algorithm, key.signing.method.algorithm), + style, + onclick := Callback.onClick(_ ⇒ showKeyDialog(key, kp.regionSet, kp.forEncryption, kp.forDecryption)) + ) } val nonActiveRows = disabledKeys.map { kp ⇒ val key = kp.key - TableRow(Seq(key.id.toString, key.encryption.method.algorithm, key.signing.method.algorithm), textDecoration.`line-through`, - onclick := Callback.onClick(_ ⇒ showKeyDialog(key, kp.regionSet, kp.forEncryption, kp.forDecryption))) + TableRow( + Seq(key.id.toString, key.encryption.method.algorithm, key.signing.method.algorithm), + textDecoration.`line-through`, + onclick := Callback.onClick(_ ⇒ showKeyDialog(key, kp.regionSet, kp.forEncryption, kp.forDecryption)) + ) } activeRows ++ nonActiveRows @@ -73,7 +79,11 @@ class KeysView()(implicit context: AppContext, kc: KeysContext, rc: RegionContex val propsRx = Var("") Modal() .withTitle(context.locale.generateKey) - .withBody(Form(FormInput.textArea(context.locale.config, placeholder := propsPlaceholder, rows := 20, AppComponents.tabOverride, propsRx.reactiveInput))) + .withBody( + Form( + FormInput.textArea(context.locale.config, placeholder := propsPlaceholder, rows := 20, AppComponents.tabOverride, propsRx.reactiveInput) + ) + ) .withButtons( AppComponents.modalSubmit(onclick := Callback.onClick(_ ⇒ doGenerate(propsRx.now))), AppComponents.modalClose() @@ -90,7 +100,8 @@ class KeysView()(implicit context: AppContext, kc: KeysContext, rc: RegionContex } private[this] def showExportDialog(key: KeySet): Unit = { - AppComponents.exportDialog(context.locale.exportKey, s"${key.id}.json", ExportUtils.encodeKey(key)) + AppComponents + .exportDialog(context.locale.exportKey, s"${key.id}.json", ExportUtils.encodeKey(key)) .show(backdrop = false) } @@ -100,14 +111,14 @@ class KeysView()(implicit context: AppContext, kc: KeysContext, rc: RegionContex } def renderPermissions() = { - val forEncryptionRx = Var(forEncryption) - val forDecryptionRx = Var(forDecryption) - val allIds = rc.regions.now.regions.keys.toVector + val forEncryptionRx = Var(forEncryption) + val forDecryptionRx = Var(forDecryption) + val allIds = rc.regions.now.regions.keys.toVector val (idSelect, idSelectRendered) = AppComponents.idSelect(context.locale.regions, allIds, regionSet.toVector) Rx((forEncryptionRx(), forDecryptionRx(), idSelect.selected())) .triggerLater(updatePermissions(idSelect.selected.now.toSet, forEncryptionRx.now, forDecryptionRx.now)) - + Form( FormInput.checkbox(context.locale.keyForEncryption, forEncryptionRx.reactiveInput), FormInput.checkbox(context.locale.keyForDecryption, forDecryptionRx.reactiveInput), @@ -131,4 +142,3 @@ class KeysView()(implicit context: AppContext, kc: KeysContext, rc: RegionContex .show() } } - diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/metadata/EmbeddedResourceView.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/metadata/EmbeddedResourceView.scala index 9fcd5a85..a4a36ae8 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/metadata/EmbeddedResourceView.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/metadata/EmbeddedResourceView.scala @@ -16,17 +16,21 @@ object EmbeddedResourceView { } class EmbeddedResourceView(path: String, metadatas: Seq[Metadata])(implicit context: AppContext) extends BootstrapHtmlComponent { - val opened = Var(false) + val opened = Var(false) val selectedMetadata = Var(None: Option[Metadata]) def renderTag(md: ModifierT*): TagT = { - val returnToTableLink = AppComponents.iconLink(context.locale.close, AppIcons.close, - Bootstrap.textStyle.danger, fontWeight.bold, onclick := Callback.onClick(_ ⇒ selectedMetadata() = None)) + val returnToTableLink = AppComponents.iconLink( + context.locale.close, + AppIcons.close, + Bootstrap.textStyle.danger, + fontWeight.bold, + onclick := Callback.onClick(_ ⇒ selectedMetadata() = None) + ) div( // Link AppComponents.dropdownLink(s"$path (${context.locale.entries(metadatas.length)})", opened), - // Content Rx[Frag] { if (opened()) { diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/metadata/EmbeddedResourcesView.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/metadata/EmbeddedResourcesView.scala index 88d57dc1..08556ff9 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/metadata/EmbeddedResourcesView.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/metadata/EmbeddedResourcesView.scala @@ -29,4 +29,3 @@ class EmbeddedResourcesView(resources: Metadata.EmbeddedResources)(implicit cont ) } } - diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/metadata/FileListView.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/metadata/FileListView.scala index 3205cf1b..2d43fd88 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/metadata/FileListView.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/metadata/FileListView.scala @@ -22,13 +22,14 @@ class FileListView(files: Metadata.FileList)(implicit context: AppContext) exten context.locale.modifiedDate ) val rows = files.files.map { file ⇒ - TableRow(Seq( - file.path.mkString(Path.Delimiter), - MemorySize.toString(file.size), - context.timeFormat.timestamp(file.timestamp) - )) + TableRow( + Seq( + file.path.mkString(Path.Delimiter), + MemorySize.toString(file.size), + context.timeFormat.timestamp(file.timestamp) + ) + ) } - PagedTable.static(tableHeading, rows, 10).renderTag(md:_*) + PagedTable.static(tableHeading, rows, 10).renderTag(md: _*) } } - diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/metadata/MetadataListView.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/metadata/MetadataListView.scala index eb03d549..0772bcb0 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/metadata/MetadataListView.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/metadata/MetadataListView.scala @@ -1,6 +1,5 @@ package com.karasiq.shadowcloud.webapp.components.metadata - import com.karasiq.bootstrap.Bootstrap.default._ import com.karasiq.shadowcloud.metadata.Metadata import com.karasiq.shadowcloud.model.{File, RegionId} @@ -38,12 +37,13 @@ object MetadataListView { } } -final class MetadataListView(regionId: RegionId, file: File, available: Rx[Set[Metadata.Tag.Disposition]])(implicit context: AppContext) extends BootstrapHtmlComponent { +final class MetadataListView(regionId: RegionId, file: File, available: Rx[Set[Metadata.Tag.Disposition]])(implicit context: AppContext) + extends BootstrapHtmlComponent { import MetadataListView.utils def renderTag(md: ModifierT*): TagT = { def renderDisposition(disposition: Metadata.Tag.Disposition): Tag = { - val opened = Var(false) + val opened = Var(false) val isAvailable = available.map(_.contains(disposition)) div( isAvailable.reactiveShow, @@ -64,4 +64,3 @@ final class MetadataListView(regionId: RegionId, file: File, available: Rx[Set[M ) } } - diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/metadata/MetadataTable.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/metadata/MetadataTable.scala index 21756bc7..a3c3099d 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/metadata/MetadataTable.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/metadata/MetadataTable.scala @@ -93,13 +93,16 @@ final class MetadataTable(metadatas: Seq[Metadata])(implicit controller: Metadat appContext.locale.content ) - val rows = metadatas.map(metadata ⇒ TableRow(Seq( - utils.metadataTagString(metadata), - utils.metadataTypeString(metadata), - utils.metadataContent(metadata), - ))) + val rows = metadatas.map(metadata ⇒ + TableRow( + Seq( + utils.metadataTagString(metadata), + utils.metadataTypeString(metadata), + utils.metadataContent(metadata) + ) + ) + ) PagedTable.static(heading, rows) } } - diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/metadata/MetadataView.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/metadata/MetadataView.scala index 9269bc95..2b1c38e5 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/metadata/MetadataView.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/metadata/MetadataView.scala @@ -1,7 +1,5 @@ package com.karasiq.shadowcloud.webapp.components.metadata - - import com.karasiq.bootstrap.Bootstrap.default._ import com.karasiq.shadowcloud.metadata.Metadata import com.karasiq.shadowcloud.webapp.context.AppContext diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/metadata/TableView.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/metadata/TableView.scala index 66cd5bdc..e85e9267 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/metadata/TableView.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/metadata/TableView.scala @@ -23,7 +23,6 @@ class TableView(table: Metadata.Table)(implicit context: AppContext) extends Boo TableRow(Seq(name, values.mkString(", "))) } - PagedTable.static(heading, rows, 10).renderTag(md:_*) + PagedTable.static(heading, rows, 10).renderTag(md: _*) } } - diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/metadata/TextView.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/metadata/TextView.scala index 95e03695..22dbc484 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/metadata/TextView.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/metadata/TextView.scala @@ -22,4 +22,3 @@ class TextView(text: Metadata.Text)(implicit context: AppContext) extends Bootst } } } - diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/metadata/ThumbnailImage.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/metadata/ThumbnailImage.scala index 7f0b8bf5..0fbcabd2 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/metadata/ThumbnailImage.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/metadata/ThumbnailImage.scala @@ -14,9 +14,8 @@ object ThumbnailImage { class ThumbnailImage(thumbnail: Metadata.Thumbnail) extends BootstrapHtmlComponent { def renderTag(md: ModifierT*): TagT = { - val blob = Blobs.fromBytes(thumbnail.data.toArray) + val blob = Blobs.fromBytes(thumbnail.data.toArray) val imageUrl = Blobs.getUrl(blob) img(src := imageUrl, alt := "Preview image") } } - diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/region/ExportImportModal.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/region/ExportImportModal.scala index 88145faf..bad3a4d9 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/region/ExportImportModal.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/region/ExportImportModal.scala @@ -27,41 +27,40 @@ object ExportImportModal { keyChain ← context.api.getKeys() } yield (snapshot, keyChain) - future.foreach { - case (snapshot, keyChain) ⇒ - val (storagesSelect, storagesSelectRendered) = AppComponents.idSelect(context.locale.storages, snapshot.storages.keys) - val (regionsSelect, regionsSelectRendered) = AppComponents.idSelect(context.locale.regions, snapshot.regions.keys) - val (keysSelect, keysSelectRendered) = AppComponents.idSelect(context.locale.keys, keyChain.keys.map(_.key.id.toString)) + future.foreach { case (snapshot, keyChain) ⇒ + val (storagesSelect, storagesSelectRendered) = AppComponents.idSelect(context.locale.storages, snapshot.storages.keys) + val (regionsSelect, regionsSelectRendered) = AppComponents.idSelect(context.locale.regions, snapshot.regions.keys) + val (keysSelect, keysSelectRendered) = AppComponents.idSelect(context.locale.keys, keyChain.keys.map(_.key.id.toString)) - Modal(context.locale.export) - .withBody( - Form( - storagesSelectRendered, - regionsSelectRendered, - keysSelectRendered - ) + Modal(context.locale.export) + .withBody( + Form( + storagesSelectRendered, + regionsSelectRendered, + keysSelectRendered ) - .withButtons(AppComponents.modalSubmit(onclick := Callback.onClick { _ ⇒ - val regions = regionsSelect.selected.now.toSet - val storages = storagesSelect.selected.now.toSet - val keys = keysSelect.selected.now.toSet + ) + .withButtons(AppComponents.modalSubmit(onclick := Callback.onClick { _ ⇒ + val regions = regionsSelect.selected.now.toSet + val storages = storagesSelect.selected.now.toSet + val keys = keysSelect.selected.now.toSet - val filteredSnapshot = snapshot.copy( - snapshot.regions.collect { - case (regionId, status) if regions(regionId) ⇒ - regionId → status.copy(storages = status.storages) - }, - snapshot.storages.collect { - case (storageId, status) if storages(storageId) ⇒ - storageId → status.copy(regions = status.regions) - } - ) + val filteredSnapshot = snapshot.copy( + snapshot.regions.collect { + case (regionId, status) if regions(regionId) ⇒ + regionId → status.copy(storages = status.storages) + }, + snapshot.storages.collect { + case (storageId, status) if storages(storageId) ⇒ + storageId → status.copy(regions = status.regions) + } + ) - val filteredKeys = keyChain.copy(keyChain.keys.filter(k ⇒ keys(k.key.id.toString))) - val json = Json.prettyPrint(Json.toJson(ExportEntity(filteredSnapshot, filteredKeys))) - AppComponents.exportDialog(context.locale.export, "sc-export.json", json).show() - })) - .show() + val filteredKeys = keyChain.copy(keyChain.keys.filter(k ⇒ keys(k.key.id.toString))) + val json = Json.prettyPrint(Json.toJson(ExportEntity(filteredSnapshot, filteredKeys))) + AppComponents.exportDialog(context.locale.export, "sc-export.json", json).show() + })) + .show() } } @@ -77,22 +76,20 @@ object ExportImportModal { }) def createStorages() = - Future.sequence(regions.storages.map { - case (id, storage) ⇒ - for { - st ← context.api.createStorage(id, storage.storageProps) - _ ← if (storage.suspended) context.api.suspendStorage(id) else Future.successful(()) - } yield st + Future.sequence(regions.storages.map { case (id, storage) ⇒ + for { + st ← context.api.createStorage(id, storage.storageProps) + _ ← if (storage.suspended) context.api.suspendStorage(id) else Future.successful(()) + } yield st }) def createRegions() = - Future.sequence(regions.regions.map { - case (id, region) ⇒ - for { - st ← context.api.createRegion(id, region.regionConfig) - _ ← context.api.suspendRegion(id) // Always suspend - _ ← Future.sequence(region.storages.map(storageId ⇒ context.api.registerStorage(id, storageId))) - } yield st + Future.sequence(regions.regions.map { case (id, region) ⇒ + for { + st ← context.api.createRegion(id, region.regionConfig) + _ ← context.api.suspendRegion(id) // Always suspend + _ ← Future.sequence(region.storages.map(storageId ⇒ context.api.registerStorage(id, storageId))) + } yield st }) for { diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/region/HealthView.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/region/HealthView.scala index 62f2aa45..6b96058b 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/region/HealthView.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/region/HealthView.scala @@ -29,12 +29,11 @@ class HealthView(healthStatusRx: Rx[HealthStatus])(implicit context: AppContext) // div(b(context.locale.freeSpace, ": ", MemorySize.toString(storageHealth.freeSpace))), div(b(context.locale.usedSpace, ": ", MemorySize.toString(hs.usedSpace))), div(b(context.locale.totalSpace, ": ", MemorySize.toString(hs.totalSpace))), - - ProgressBar.basic(Var(HealthStatus.getUsedPercentage(hs))) + ProgressBar + .basic(Var(HealthStatus.getUsedPercentage(hs))) .renderTag(progressBarStyle, ProgressBarStyle.striped) ) } div(progressBarRx, healthStatusRx.map(_.totalSpace == 0).reactiveHide) } } - diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/region/IndexScopeSelector.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/region/IndexScopeSelector.scala index 7efd891d..5c3ee15b 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/region/IndexScopeSelector.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/region/IndexScopeSelector.scala @@ -22,7 +22,7 @@ object IndexScopeSelector { } class IndexScopeSelector(implicit context: AppContext) extends BootstrapHtmlComponent { - val opened = Var(false) + val opened = Var(false) val dateInput = DateInput(context.locale.indexSnapshotDate) val selectedScope = Rx[IndexScope] { @@ -42,9 +42,9 @@ class IndexScopeSelector(implicit context: AppContext) extends BootstrapHtmlComp def renderTag(md: ModifierT*): TagT = { val link = Rx { val isOpened = opened() - val icon = if (isOpened) AppIcons.historyScope else AppIcons.currentScope - val style = if (isOpened) Bootstrap.textStyle.warning else Bootstrap.textStyle.success - val title = if (isOpened) context.locale.historyScope else context.locale.currentScope + val icon = if (isOpened) AppIcons.historyScope else AppIcons.currentScope + val style = if (isOpened) Bootstrap.textStyle.warning else Bootstrap.textStyle.success + val title = if (isOpened) context.locale.historyScope else context.locale.currentScope AppComponents.iconLink(title, icon, style, onclick := Callback.onClick(_ ⇒ opened() = !opened.now)) } @@ -57,4 +57,3 @@ class IndexScopeSelector(implicit context: AppContext) extends BootstrapHtmlComp ) } } - diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/region/RegionSelector.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/region/RegionSelector.scala index 1e778b7a..15615147 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/region/RegionSelector.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/region/RegionSelector.scala @@ -17,13 +17,14 @@ object RegionSelector { private def getRegionIds(state: RegionStateReport): Seq[RegionId] = { state.regions .filterNot(_._2.suspended) - .keys.toSeq + .keys + .toSeq .sorted } } class RegionSelector(implicit context: AppContext, regionContext: RegionContext) extends BootstrapHtmlComponent { - val selectedRegion = Var(RegionSelector.getRegionIds(regionContext.regions.now).headOption) + val selectedRegion = Var(RegionSelector.getRegionIds(regionContext.regions.now).headOption) private[this] val selectField = renderSelectField() selectedRegion.triggerLater { @@ -45,7 +46,8 @@ class RegionSelector(implicit context: AppContext, regionContext: RegionContext) private[this] def renderSelectField(): FormSelect = { val options = regionContext.regions.map { state ⇒ - RegionSelector.getRegionIds(state) + RegionSelector + .getRegionIds(state) .map(id ⇒ FormSelectOption(id, id)) } diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/region/RegionSwitcher.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/region/RegionSwitcher.scala index 594acc88..b16c17c3 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/region/RegionSwitcher.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/region/RegionSwitcher.scala @@ -14,11 +14,10 @@ object RegionSwitcher { class RegionSwitcher(implicit context: AppContext, regionContext: RegionContext) extends BootstrapHtmlComponent { val regionSelector = RegionSelector() - val scopeSelector = IndexScopeSelector() + val scopeSelector = IndexScopeSelector() def renderTag(md: ModifierT*): TagT = { - Button(ButtonStyle.warning)(AppIcons.region, context.locale.region, - onclick := Callback.onClick(_ ⇒ this.show()), md) + Button(ButtonStyle.warning)(AppIcons.region, context.locale.region, onclick := Callback.onClick(_ ⇒ this.show()), md) } def renderModal(): Modal = { @@ -30,9 +29,7 @@ class RegionSwitcher(implicit context: AppContext, regionContext: RegionContext) Modal(context.locale.region, content, AppComponents.modalClose()) } - def show(): Unit = { renderModal().show() } } - diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/region/RegionsStoragesPanel.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/region/RegionsStoragesPanel.scala index 100b6e35..b86b1cca 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/region/RegionsStoragesPanel.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/region/RegionsStoragesPanel.scala @@ -21,14 +21,12 @@ class RegionsStoragesPanel(implicit context: AppContext, regionContext: RegionCo RegionsView(), GridSystem.col.responsive(12, 12, 6, 4) ), - div( h3(context.locale.storages), hr, StoragesView(), GridSystem.col.responsive(12, 12, 6, 4) ), - div( h3(context.locale.keys), hr, @@ -38,4 +36,3 @@ class RegionsStoragesPanel(implicit context: AppContext, regionContext: RegionCo ) } } - diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/region/RegionsView.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/region/RegionsView.scala index ed17cb9b..462e1ba4 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/region/RegionsView.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/region/RegionsView.scala @@ -36,13 +36,11 @@ object RegionsView { class RegionsView(implicit context: AppContext, regionContext: RegionContext, keysContext: KeysContext) extends BootstrapHtmlComponent { def renderTag(md: ModifierT*): TagT = { - val regionViewsRx = regionContext.regions.fold(Map.empty[RegionId, Tag]) { - case (views, report) ⇒ - val newMap = report.regions.map { - case (regionId, _) ⇒ - regionId → views.getOrElse(regionId, renderRegion(regionId)) - } - newMap + val regionViewsRx = regionContext.regions.fold(Map.empty[RegionId, Tag]) { case (views, report) ⇒ + val newMap = report.regions.map { case (regionId, _) ⇒ + regionId → views.getOrElse(regionId, renderRegion(regionId)) + } + newMap } div( @@ -86,9 +84,12 @@ class RegionsView(implicit context: AppContext, regionContext: RegionContext, ke // Utils.toSafeIdentifier(newRegionNameRx.now) doCreate(newRegionIdRx.now) }), - Button(ButtonStyle.info)(context.locale.uniqueRegionId, onclick := Callback.onClick { _ ⇒ - newRegionIdRx() = RegionsView.uniqueRegionId(newRegionIdRx.now) - }), + Button(ButtonStyle.info)( + context.locale.uniqueRegionId, + onclick := Callback.onClick { _ ⇒ + newRegionIdRx() = RegionsView.uniqueRegionId(newRegionIdRx.now) + } + ), AppComponents.modalClose() ) .show() @@ -121,9 +122,9 @@ class RegionsView(implicit context: AppContext, regionContext: RegionContext, ke Button(ButtonStyle.danger, ButtonSize.small, block = true)( AppIcons.suspend, context.locale.suspend, - onclick := Callback.onClick { _ => + onclick := Callback.onClick { _ ⇒ val future = Future.sequence(regionContext.regions.now.regions.keys.map(context.api.suspendRegion)) - future.onComplete(_ => regionContext.updateAll()) + future.onComplete(_ ⇒ regionContext.updateAll()) } ) } diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/region/StorageConfigView.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/region/StorageConfigView.scala index 38759e50..a60c087f 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/region/StorageConfigView.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/region/StorageConfigView.scala @@ -76,7 +76,7 @@ class StorageConfigView(storageId: StorageId)(implicit context: AppContext, regi Button(ButtonStyle.danger, ButtonSize.extraSmall)(AppIcons.delete, context.locale.delete, onclick := Callback.onClick(_ ⇒ doDelete())) val resetButton = - Button(ButtonStyle.info, ButtonSize.extraSmall)(AppIcons.refresh, context.locale.reset, onclick := Callback.onClick(_ => doReset())) + Button(ButtonStyle.info, ButtonSize.extraSmall)(AppIcons.refresh, context.locale.reset, onclick := Callback.onClick(_ ⇒ doReset())) ButtonGroup(ButtonGroupSize.extraSmall, suspendButton, deleteButton, resetButton) } diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/region/StoragesView.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/region/StoragesView.scala index 07a933a1..685ba424 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/region/StoragesView.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/region/StoragesView.scala @@ -48,7 +48,7 @@ class StoragesView(implicit context: AppContext, regionContext: RegionContext) e def showCreateDialog() = { val newStorageIdRx = Var(StoragesView.newStorageId()) - val propsRx = Var("") + val propsRx = Var("") val storageTypeSelect = { val storageTypesRx = context.api.getStorageTypes().toRx(Set.empty).map(_.toSeq.sorted) @@ -57,22 +57,24 @@ class StoragesView(implicit context: AppContext, regionContext: RegionContext) e storageTypeSelect.selected.map(_.headOption).foreach { case Some(storageType) ⇒ - context.api.getDefaultStorageConfig(storageType) + context.api + .getDefaultStorageConfig(storageType) .map(_.data.utf8String) .foreach(propsRx.update) case None ⇒ - // Ignore + // Ignore } Modal() .withTitle(context.locale.createStorage) - .withBody(Form( - FormInput.text(context.locale.storageId, newStorageIdRx.reactiveInput), - storageTypeSelect, - FormInput.textArea(context.locale.config, rows := 10, Rx(placeholder := propsRx()).auto, - AppComponents.tabOverride, propsRx.reactiveInput) - )) + .withBody( + Form( + FormInput.text(context.locale.storageId, newStorageIdRx.reactiveInput), + storageTypeSelect, + FormInput.textArea(context.locale.config, rows := 10, Rx(placeholder := propsRx()).auto, AppComponents.tabOverride, propsRx.reactiveInput) + ) + ) .withButtons( AppComponents.modalSubmit(onclick := Callback.onClick(_ ⇒ doCreate(newStorageIdRx.now, propsRx.now))), AppComponents.modalClose() @@ -81,7 +83,8 @@ class StoragesView(implicit context: AppContext, regionContext: RegionContext) e } Button(ButtonStyle.primary, ButtonSize.small, block = true)( - AppIcons.create, context.locale.createStorage, + AppIcons.create, + context.locale.createStorage, onclick := Callback.onClick(_ ⇒ showCreateDialog()) ) } @@ -93,4 +96,3 @@ class StoragesView(implicit context: AppContext, regionContext: RegionContext) e } } } - diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/themes/ThemeSelector.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/themes/ThemeSelector.scala index 8e6ce507..dc2599c9 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/themes/ThemeSelector.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/components/themes/ThemeSelector.scala @@ -1,7 +1,5 @@ package com.karasiq.shadowcloud.webapp.components.themes - - import com.karasiq.bootstrap.Bootstrap.default._ import com.karasiq.shadowcloud.webapp.utils.StorageBind import rx.{Rx, Var} @@ -9,8 +7,23 @@ import scalaTags.all._ object ThemeSelector { val Themes = Vector( - "Default", "Cerulean", "Cosmo", "Cyborg", "Darkly", "Flatly", "Journal", "Lumen", "Paper", "Readable", - "Sandstone", "Simplex", "Slate", "Spacelab", "Superhero", "United", "Yeti" + "Default", + "Cerulean", + "Cosmo", + "Cyborg", + "Darkly", + "Flatly", + "Journal", + "Lumen", + "Paper", + "Readable", + "Sandstone", + "Simplex", + "Slate", + "Spacelab", + "Superhero", + "United", + "Yeti" ) lazy val CurrentTheme = StorageBind.LS("bootstrap-theme", Themes.head) @@ -20,8 +33,7 @@ object ThemeSelector { } } -class ThemeSelector(themes: IndexedSeq[String], currentTheme: Var[String]) - extends BootstrapHtmlComponent { +class ThemeSelector(themes: IndexedSeq[String], currentTheme: Var[String]) extends BootstrapHtmlComponent { def setTheme(theme: String): Unit = { currentTheme() = theme @@ -29,7 +41,7 @@ class ThemeSelector(themes: IndexedSeq[String], currentTheme: Var[String]) def nextTheme(): Unit = { val currentIndex = themes.indexOf(currentTheme.now) - val nextIndex = (currentIndex + 1) % themes.length + val nextIndex = (currentIndex + 1) % themes.length setTheme(themes(nextIndex)) } diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/context/AppContext.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/context/AppContext.scala index 91771f56..0b6da715 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/context/AppContext.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/context/AppContext.scala @@ -15,8 +15,8 @@ import com.karasiq.taboverridejs.TabOverride object AppContext { implicit val JsExecutionContext = scala.scalajs.concurrent.JSExecutionContext.Implicits.queue - val BootstrapContext = Bootstrap.default - val CssSettings = scalacss.devOrProdDefaults + val BootstrapContext = Bootstrap.default + val CssSettings = scalacss.devOrProdDefaults def apply(): AppContext = { new DefaultAppContext() @@ -52,8 +52,8 @@ trait AppContext { } class DefaultAppContext extends AppContext { - lazy val api = AjaxApi - lazy val locale: AppLocale = AppLocale.forCode(Option(dom.window.navigator.language).getOrElse("en")) - lazy val timeFormat: TimeFormat = TimeFormat.forLocale("en" /* locale.languageCode*/) - lazy val styles = Seq(FolderTreeStyles) + lazy val api = AjaxApi + lazy val locale: AppLocale = AppLocale.forCode(Option(dom.window.navigator.language).getOrElse("en")) + lazy val timeFormat: TimeFormat = TimeFormat.forLocale("en" /* locale.languageCode*/ ) + lazy val styles = Seq(FolderTreeStyles) } diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/context/FolderContext.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/context/FolderContext.scala index fabd5cdb..55b36c3f 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/context/FolderContext.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/context/FolderContext.scala @@ -21,9 +21,9 @@ object FolderContext { def apply(_regionId: RegionId): FolderContext = { new FolderContext { val regionId = _regionId - val scope = Var(IndexScope.default) + val scope = Var(IndexScope.default) val selected = Var(Path.root) - val updates = Var(Path.root, 0) + val updates = Var(Path.root, 0) } } } diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/controllers/FileController.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/controllers/FileController.scala index d15301aa..462ced98 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/controllers/FileController.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/controllers/FileController.scala @@ -11,22 +11,26 @@ trait FileController { } object FileController { - def apply(onAddFile: File ⇒ Unit, - onDeleteFile: File ⇒ Unit, - onUpdateFile: (File, File) ⇒ Unit, - onRenameFile: (File, String) ⇒ Unit): FileController = { + def apply( + onAddFile: File ⇒ Unit, + onDeleteFile: File ⇒ Unit, + onUpdateFile: (File, File) ⇒ Unit, + onRenameFile: (File, String) ⇒ Unit + ): FileController = { new FileController { - def addFile(file: File): Unit = onAddFile(file) - def deleteFile(file: File): Unit = onDeleteFile(file) + def addFile(file: File): Unit = onAddFile(file) + def deleteFile(file: File): Unit = onDeleteFile(file) def updateFile(oldFile: File, newFile: File): Unit = onUpdateFile(oldFile, newFile) - def renameFile(file: File, newName: String): Unit = onRenameFile(file, newName) + def renameFile(file: File, newName: String): Unit = onRenameFile(file, newName) } } - def inherit(onAddFile: File ⇒ Unit = _ ⇒ (), - onDeleteFile: File ⇒ Unit = _ ⇒ (), - onUpdateFile: (File, File) ⇒ Unit = (_, _) ⇒ (), - onRenameFile: (File, String) ⇒ Unit = (_, _) ⇒ ())(implicit fc: FileController): FileController = { + def inherit( + onAddFile: File ⇒ Unit = _ ⇒ (), + onDeleteFile: File ⇒ Unit = _ ⇒ (), + onUpdateFile: (File, File) ⇒ Unit = (_, _) ⇒ (), + onRenameFile: (File, String) ⇒ Unit = (_, _) ⇒ () + )(implicit fc: FileController): FileController = { apply( file ⇒ { fc.addFile(file); onAddFile(file) }, file ⇒ { fc.deleteFile(file); onDeleteFile(file) }, diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/controllers/FolderController.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/controllers/FolderController.scala index 727f02c9..ff3c19c3 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/controllers/FolderController.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/controllers/FolderController.scala @@ -10,20 +10,17 @@ trait FolderController extends HasKeyUpdate[Path] { } object FolderController { - def apply(onUpdate: Path ⇒ Unit, - onAddFolder: Folder ⇒ Unit, - onDeleteFolder: Folder ⇒ Unit): FolderController = { + def apply(onUpdate: Path ⇒ Unit, onAddFolder: Folder ⇒ Unit, onDeleteFolder: Folder ⇒ Unit): FolderController = { new FolderController { - def update(path: Path): Unit = onUpdate(path) - def addFolder(folder: Folder): Unit = onAddFolder(folder) + def update(path: Path): Unit = onUpdate(path) + def addFolder(folder: Folder): Unit = onAddFolder(folder) def deleteFolder(folder: Folder): Unit = onDeleteFolder(folder) } } - def inherit(onUpdate: Path ⇒ Unit = _ ⇒ (), - onAddFolder: Folder ⇒ Unit = _ ⇒ (), - onDeleteFolder: Folder ⇒ Unit = _ ⇒ ()) - (implicit fc: FolderController): FolderController = apply( + def inherit(onUpdate: Path ⇒ Unit = _ ⇒ (), onAddFolder: Folder ⇒ Unit = _ ⇒ (), onDeleteFolder: Folder ⇒ Unit = _ ⇒ ())(implicit + fc: FolderController + ): FolderController = apply( path ⇒ { fc.update(path); onUpdate(path) }, folder ⇒ { fc.addFolder(folder); onAddFolder(folder) }, folder ⇒ { fc.deleteFolder(folder); onDeleteFolder(folder) } @@ -37,4 +34,4 @@ object FolderController { }, folder ⇒ folderContext.update(folder.path.parent) ) -} \ No newline at end of file +} diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/locales/AppLocale.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/locales/AppLocale.scala index 74028e50..31695853 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/locales/AppLocale.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/locales/AppLocale.scala @@ -132,4 +132,4 @@ object AppLocale { val langCode = code.toLowerCase.split("-", 2).headOption Locales.all.find(lc ⇒ langCode.contains(lc.languageCode)).getOrElse(Locales.default) } -} +} diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/locales/impl/English.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/locales/impl/English.scala index b3e5a076..90fb6793 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/locales/impl/English.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/locales/impl/English.scala @@ -6,32 +6,32 @@ import com.karasiq.shadowcloud.webapp.locales.AppLocale private[locales] object English extends AppLocale { val languageCode = "en" - val name = "Name" - val path = "Path" - val value = "Value" - val config = "Config" - val size = "Size" - val empty = "Empty" - val show = "Show" - val edit = "Edit" - val submit = "Submit" - val cancel = "Cancel" - val close = "Close" - val unknown = "Unknown" - val total = "Total" + val name = "Name" + val path = "Path" + val value = "Value" + val config = "Config" + val size = "Size" + val empty = "Empty" + val show = "Show" + val edit = "Edit" + val submit = "Submit" + val cancel = "Cancel" + val close = "Close" + val unknown = "Unknown" + val total = "Total" def entries(value: Long): String = value + " entries" - val file = "File" + val file = "File" val folder = "Folder" val delete = "Delete" - val move = "Move" - val copy = "Copy" + val move = "Move" + val copy = "Copy" val rename = "Rename" - val reset = "Reset" + val reset = "Reset" - val rootPath = "(Root)" - val emptyPath = "(Empty)" + val rootPath = "(Root)" + val emptyPath = "(Empty)" val createFolder = "Create folder" val deleteFolder = "Delete folder" @@ -39,89 +39,89 @@ private[locales] object English extends AppLocale { val audio = "Audio" val video = "Video" - val uploadFiles = "Upload" + val uploadFiles = "Upload" val downloadFile = "Download" - val deleteFile = "Delete" - val repairFile = "Replicate" - val playFile = "Play" + val deleteFile = "Delete" + val repairFile = "Replicate" + val playFile = "Play" val viewTextFile = "View text" - val inspectFile = "Inspect" - val pasteText = "Paste text" - val changeView = "Change view" + val inspectFile = "Inspect" + val pasteText = "Paste text" + val changeView = "Change view" - val fileId = "File ID" - val createdDate = "Created" + val fileId = "File ID" + val createdDate = "Created" val modifiedDate = "Last modified" - val revision = "Revision" + val revision = "Revision" - val preview = "Preview" + val preview = "Preview" val metadata = "Metadata" - val content = "Content" + val content = "Content" val availability = "Availability" - val revisions = "Revisions" + val revisions = "Revisions" val foldersView = "Folders" val regionsView = "Regions" - val logs = "Logs" - val challenges = "Challenges" - - val storage = "Storage" - val storages = "Storages" - val region = "Region" - val regions = "Regions" - val indexScope = "Index scope" - val currentScope = "Current" - val historyScope = "History" + val logs = "Logs" + val challenges = "Challenges" + + val storage = "Storage" + val storages = "Storages" + val region = "Region" + val regions = "Regions" + val indexScope = "Index scope" + val currentScope = "Current" + val historyScope = "History" val indexSnapshotDate = "Snapshot date" val collectGarbage = "Collect garbage" - val compactIndex = "Compact index" - val repairRegion = "Replicate region" - val synchronize = "Synchronize" + val compactIndex = "Compact index" + val repairRegion = "Replicate region" + val synchronize = "Synchronize" val writableSpace = "Writable space" - val totalSpace = "Total space" - val freeSpace = "Free space" - val usedSpace = "Used space" - - val regionId = "Region ID" - val regionIdHint = "Hint: format [a-z0-9_-] is recommended" - val uniqueRegionId = "Unique ID" - val storageId = "Storage ID" - val storageType = "Storage type" - val createRegion = "Create region" - val createStorage = "Create storage" - val registerStorage = "Register storage" - val registerRegion = "Register region" + val totalSpace = "Total space" + val freeSpace = "Free space" + val usedSpace = "Used space" + + val regionId = "Region ID" + val regionIdHint = "Hint: format [a-z0-9_-] is recommended" + val uniqueRegionId = "Unique ID" + val storageId = "Storage ID" + val storageType = "Storage type" + val createRegion = "Create region" + val createStorage = "Create storage" + val registerStorage = "Register storage" + val registerRegion = "Register region" val unregisterStorage = "Unregister storage" - val unregisterRegion = "Unregister region" - val suspend = "Suspend" - val resume = "Resume" - val export = "Export" - val `import` = "Import" - - val keys = "Keys" - val keyId = "Key ID" - val keyEncAlgorithm = "Encryption algorithm" + val unregisterRegion = "Unregister region" + val suspend = "Suspend" + val resume = "Resume" + val export = "Export" + val `import` = "Import" + + val keys = "Keys" + val keyId = "Key ID" + val keyEncAlgorithm = "Encryption algorithm" val keySignAlgorithm = "Signature algorithm" - val keyPermissions = "Key permissions" + val keyPermissions = "Key permissions" val keyForEncryption = "Encryption" val keyForDecryption = "Decryption" - val generateKey = "Generate key" - val exportKey = "Export key" - val importKey = "Import key" + val generateKey = "Generate key" + val exportKey = "Export key" + val importKey = "Import key" - val metadataParser = "Parser" + val metadataParser = "Parser" val metadataDisposition = "Disposition" - val metadataType = "Type" - val metadataFormat = "Format" - - val metadataText = "Text" - val metadataTable = "Table" - val metadataFileList = "File list" - val metadataThumbnail = "Thumbnail" - val metadataImageData = "Image data" + val metadataType = "Type" + val metadataFormat = "Format" + + val metadataText = "Text" + val metadataTable = "Table" + val metadataFileList = "File list" + val metadataThumbnail = "Thumbnail" + val metadataImageData = "Image data" val metadataEmbeddedResources = "Embedded resources" def deleteFolderConfirmation(path: Path): String = { diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/locales/impl/Locales.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/locales/impl/Locales.scala index 30d7b484..be2cc66c 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/locales/impl/Locales.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/locales/impl/Locales.scala @@ -3,6 +3,6 @@ package com.karasiq.shadowcloud.webapp.locales.impl import com.karasiq.shadowcloud.webapp.locales.AppLocale private[locales] object Locales { - val all = Vector(English, Russian) + val all = Vector(English, Russian) val default: AppLocale = all.head } diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/locales/impl/Russian.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/locales/impl/Russian.scala index 6f49acda..04376f24 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/locales/impl/Russian.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/locales/impl/Russian.scala @@ -6,32 +6,32 @@ import com.karasiq.shadowcloud.webapp.locales.AppLocale private[locales] object Russian extends AppLocale { val languageCode = "ru" - val name = "Имя" - val path = "Путь" - val value = "Значение" - val config = "Настройки" - val size = "Размер" - val empty = "Пусто" - val show = "Показать" - val edit = "Редактировать" - val submit = "Отправить" - val cancel = "Отмена" - val close = "Закрыть" - val unknown = "Неизвестно" - val total = "Всего" + val name = "Имя" + val path = "Путь" + val value = "Значение" + val config = "Настройки" + val size = "Размер" + val empty = "Пусто" + val show = "Показать" + val edit = "Редактировать" + val submit = "Отправить" + val cancel = "Отмена" + val close = "Закрыть" + val unknown = "Неизвестно" + val total = "Всего" def entries(value: Long): String = value + " записей" - val file = "Файл" + val file = "Файл" val folder = "Папка" val delete = "Удалить" - val move = "Переместить" - val copy = "Копировать" + val move = "Переместить" + val copy = "Копировать" val rename = "Переименовать" - val reset = "Сбросить" + val reset = "Сбросить" - val rootPath = "(Корень)" - val emptyPath = "(Пусто)" + val rootPath = "(Корень)" + val emptyPath = "(Пусто)" val createFolder = "Создать папку" val deleteFolder = "Удалить папку" @@ -39,89 +39,89 @@ private[locales] object Russian extends AppLocale { val audio = "Аудио" val video = "Видео" - val uploadFiles = "Загрузить" + val uploadFiles = "Загрузить" val downloadFile = "Скачать" - val deleteFile = "Удалить" - val repairFile = "Реплицировать" - val playFile = "Просмотр" + val deleteFile = "Удалить" + val repairFile = "Реплицировать" + val playFile = "Просмотр" val viewTextFile = "Открыть текст" - val inspectFile = "Подробности" - val pasteText = "Вставить текст" - val changeView = "Изменить отображение" + val inspectFile = "Подробности" + val pasteText = "Вставить текст" + val changeView = "Изменить отображение" - val fileId = "ID файла" - val createdDate = "Создан" + val fileId = "ID файла" + val createdDate = "Создан" val modifiedDate = "Изменён" - val revision = "Ревизия" + val revision = "Ревизия" - val preview = "Предпросмотр" + val preview = "Предпросмотр" val metadata = "Метаданные" - val content = "Содержание" + val content = "Содержание" val availability = "Доступность" - val revisions = "Ревизии" + val revisions = "Ревизии" val foldersView = "Папки" val regionsView = "Регионы" - val logs = "Журнал" - val challenges = "Капча" - - val storage = "Хранилище" - val storages = "Хранилища" - val region = "Регион" - val regions = "Регионы" - val indexScope = "Контекст" - val currentScope = "Текущий" - val historyScope = "История" + val logs = "Журнал" + val challenges = "Капча" + + val storage = "Хранилище" + val storages = "Хранилища" + val region = "Регион" + val regions = "Регионы" + val indexScope = "Контекст" + val currentScope = "Текущий" + val historyScope = "История" val indexSnapshotDate = "Дата снимка" val collectGarbage = "Собрать мусор" - val compactIndex = "Сжать индекс" - val repairRegion = "Репликация" - val synchronize = "Синхронизация" + val compactIndex = "Сжать индекс" + val repairRegion = "Репликация" + val synchronize = "Синхронизация" val writableSpace = "Доступно" - val totalSpace = "Всего" - val freeSpace = "Свободно" - val usedSpace = "Занято" - - val regionId = "ID региона" - val regionIdHint = "Подсказка: рекомендуется формат [a-z0-9_-]" - val uniqueRegionId = "Уникальный ID" - val storageId = "ID хранилища" - val storageType = "Тип хранилища" - val createRegion = "Создать регион" - val createStorage = "Создать хранилище" - val registerStorage = "Зарегистрировать хранилище" - val registerRegion = "Зарегистрировать регион" + val totalSpace = "Всего" + val freeSpace = "Свободно" + val usedSpace = "Занято" + + val regionId = "ID региона" + val regionIdHint = "Подсказка: рекомендуется формат [a-z0-9_-]" + val uniqueRegionId = "Уникальный ID" + val storageId = "ID хранилища" + val storageType = "Тип хранилища" + val createRegion = "Создать регион" + val createStorage = "Создать хранилище" + val registerStorage = "Зарегистрировать хранилище" + val registerRegion = "Зарегистрировать регион" val unregisterStorage = "Разрегистрировать хранилище" - val unregisterRegion = "Разрегистрировать регион" - val suspend = "Остановить" - val resume = "Запустить" - val export = "Экспорт" - val `import` = "Импорт" - - val keys = "Ключи" - val keyId = "ID ключа" - val keyEncAlgorithm = "Алгоритм шифрования" + val unregisterRegion = "Разрегистрировать регион" + val suspend = "Остановить" + val resume = "Запустить" + val export = "Экспорт" + val `import` = "Импорт" + + val keys = "Ключи" + val keyId = "ID ключа" + val keyEncAlgorithm = "Алгоритм шифрования" val keySignAlgorithm = "Алгоритм подписи" - val keyPermissions = "Разрешения ключа" + val keyPermissions = "Разрешения ключа" val keyForEncryption = "Шифрование" val keyForDecryption = "Дешифрование" - val generateKey = "Сгенерировать ключ" - val exportKey = "Экспорт ключа" - val importKey = "Импорт ключа" + val generateKey = "Сгенерировать ключ" + val exportKey = "Экспорт ключа" + val importKey = "Импорт ключа" - val metadataParser = "Парсер" + val metadataParser = "Парсер" val metadataDisposition = "Назначение" - val metadataType = "Тип" - val metadataFormat = "Формат" - - val metadataText = "Текст" - val metadataTable = "Таблица" - val metadataFileList = "Список файлов" - val metadataThumbnail = "Миниатюра" - val metadataImageData = "Данные изображения" + val metadataType = "Тип" + val metadataFormat = "Формат" + + val metadataText = "Текст" + val metadataTable = "Таблица" + val metadataFileList = "Список файлов" + val metadataThumbnail = "Миниатюра" + val metadataImageData = "Данные изображения" val metadataEmbeddedResources = "Встроенные ресурсы" def deleteFolderConfirmation(path: Path): String = { diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/styles/FolderTreeStyles.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/styles/FolderTreeStyles.scala index 0a0bc569..874548e4 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/styles/FolderTreeStyles.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/styles/FolderTreeStyles.scala @@ -1,7 +1,5 @@ package com.karasiq.shadowcloud.webapp.styles - - import com.karasiq.shadowcloud.webapp.context.AppContext.CssSettings._ object FolderTreeStyles extends StyleSheet.Inline { diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/utils/Blobs.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/utils/Blobs.scala index 26413a33..6233af53 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/utils/Blobs.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/utils/Blobs.scala @@ -10,8 +10,7 @@ import scala.concurrent.{ExecutionContext, Future, Promise} import scala.scalajs.js import scala.scalajs.js.typedarray.{ArrayBuffer, TypedArrayBuffer, Uint8Array} -/** - * Blob/file utility +/** Blob/file utility */ object Blobs { def fromBytes(data: Array[Byte], contentType: String = ""): Blob = { @@ -33,13 +32,16 @@ object Blobs { } def saveBlob(blob: Blob, fileName: String): Unit = { - val url = URL.createObjectURL(blob) + val url = URL.createObjectURL(blob) val anchor = a(href := url, attr("download") := fileName, target := "_blank", display.none).render dom.document.body.appendChild(anchor) - dom.window.setTimeout(() ⇒ { - dom.document.body.removeChild(anchor) - URL.revokeObjectURL(url) - }, 500) + dom.window.setTimeout( + () ⇒ { + dom.document.body.removeChild(anchor) + URL.revokeObjectURL(url) + }, + 500 + ) anchor.click() } @@ -49,7 +51,7 @@ object Blobs { def toDataURL(blob: Blob): Future[String] = { val promise = Promise[String] - val reader = new FileReader + val reader = new FileReader reader.readAsDataURL(blob) reader.onloadend = (_: ProgressEvent) ⇒ { promise.success(reader.result.asInstanceOf[String]) @@ -68,7 +70,7 @@ object Blobs { def toString(blob: Blob): Future[String] = { val promise = Promise[String] - val reader = new FileReader + val reader = new FileReader reader.readAsText(blob) reader.onloadend = (_: ProgressEvent) ⇒ { promise.success(reader.result.asInstanceOf[String]) @@ -83,7 +85,7 @@ object Blobs { def toBytes(blob: Blob): Future[ByteString] = { val promise = Promise[ByteString] - val reader = new FileReader + val reader = new FileReader reader.readAsArrayBuffer(blob) reader.onloadend = (_: ProgressEvent) ⇒ { val buffer = TypedArrayBuffer.wrap(reader.result.asInstanceOf[ArrayBuffer]) diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/utils/HasUpdate.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/utils/HasUpdate.scala index db0a029e..ef2c159e 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/utils/HasUpdate.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/utils/HasUpdate.scala @@ -6,4 +6,4 @@ trait HasUpdate { trait HasKeyUpdate[K] { def update(key: K): Unit -} \ No newline at end of file +} diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/utils/HtmlUtils.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/utils/HtmlUtils.scala index 1bfe4c12..fe3687a0 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/utils/HtmlUtils.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/utils/HtmlUtils.scala @@ -9,8 +9,8 @@ object HtmlUtils { def extractContent(html: String, format: String = HtmlMime): Seq[scalajs.dom.Node] = { import com.karasiq.bootstrap.Bootstrap.default.DOMListIndexedSeq - val domParser = new DOMParser - val document = domParser.parseFromString(html, format) + val domParser = new DOMParser + val document = domParser.parseFromString(html, format) val documentBody = document.getElementsByTagName("body").headOption documentBody .map(body ⇒ body.childNodes.map(node ⇒ node)) diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/utils/LocalStorage.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/utils/LocalStorage.scala index 41a0cf3d..9bcfb52c 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/utils/LocalStorage.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/utils/LocalStorage.scala @@ -18,7 +18,7 @@ object LocalStorage { val value = Var(Option(localStorage.getItem(key)).getOrElse("")) def updateLS(): Unit = localStorage.setItem(key, value.now) value.debounce(5 seconds).triggerLater(updateLS()) - dom.window.addEventListener("unload", (_: dom.Event) => updateLS()) + dom.window.addEventListener("unload", (_: dom.Event) ⇒ updateLS()) value } ) diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/utils/RxLocation.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/utils/RxLocation.scala index babc9852..d0a75654 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/utils/RxLocation.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/utils/RxLocation.scala @@ -19,9 +19,12 @@ final class RxLocation(implicit ctx: Ctx.Owner) { window.location.hash = hash.now.fold("")("#" + _) } - dom.window.addEventListener("hashchange", { (_: Event) ⇒ - hash() = readLocationHash(window.location.hash) - }) + dom.window.addEventListener( + "hashchange", + { (_: Event) ⇒ + hash() = readLocationHash(window.location.hash) + } + ) /* jQuery(dom.window).on("hashchange", () ⇒ { hash() = readLocationHash(window.location.hash) diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/utils/RxUtils.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/utils/RxUtils.scala index d86f2930..93742a97 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/utils/RxUtils.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/utils/RxUtils.scala @@ -31,8 +31,9 @@ private[webapp] object RxUtils { } } - private[this] def createFolderRx(regionId: RegionId, pathRx: Rx[Path], scopeRx: Rx[IndexScope]) - (implicit ac: AppContext): RxWithKey[(Path, IndexScope), Folder] = { + private[this] def createFolderRx(regionId: RegionId, pathRx: Rx[Path], scopeRx: Rx[IndexScope])(implicit + ac: AppContext + ): RxWithKey[(Path, IndexScope), Folder] = { val folderRx = RxWithKey(Rx((pathRx(), scopeRx())), Folder.create(pathRx.now)) { case (path, scope) ⇒ ac.api.getFolder(regionId, path, dropChunks = true, scope) } @@ -40,15 +41,19 @@ private[webapp] object RxUtils { folderRx } - private[this] def createContextFolderRx(regionId: RegionId, pathRx: Rx[Path], scopeRx: Rx[IndexScope]) - (implicit fc: FolderContext, ac: AppContext): RxWithKey[(Path, IndexScope), Folder] = { + private[this] def createContextFolderRx(regionId: RegionId, pathRx: Rx[Path], scopeRx: Rx[IndexScope])(implicit + fc: FolderContext, + ac: AppContext + ): RxWithKey[(Path, IndexScope), Folder] = { val folderRx = createFolderRx(regionId, pathRx, scopeRx) fc.updates.filter(_._1 == pathRx.now).triggerLater(folderRx.update()) // Subscribe to updates folderRx } - private[this] def createContextFilesRx(regionId: RegionId, pathRx: Rx[Path], scopeRx: Rx[IndexScope]) - (implicit fc: FolderContext, ac: AppContext): RxWithKey[(Path, IndexScope), Set[File]] = { + private[this] def createContextFilesRx(regionId: RegionId, pathRx: Rx[Path], scopeRx: Rx[IndexScope])(implicit + fc: FolderContext, + ac: AppContext + ): RxWithKey[(Path, IndexScope), Set[File]] = { val filesRx = RxWithKey(Rx(pathRx(), scopeRx()), Set.empty[File]) { case (path, scope) ⇒ ac.api.getFiles(regionId, path, dropChunks = true, scope = scope) } diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/utils/RxWithUpdate.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/utils/RxWithUpdate.scala index 5ad75ada..e9cf78f3 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/utils/RxWithUpdate.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/utils/RxWithUpdate.scala @@ -10,10 +10,9 @@ object RxWithUpdate { } } -class RxWithUpdate[T](initial: T, getValue: T ⇒ Future[T]) - (implicit ctx: Ctx.Owner, ec: ExecutionContext) extends HasUpdate { +class RxWithUpdate[T](initial: T, getValue: T ⇒ Future[T])(implicit ctx: Ctx.Owner, ec: ExecutionContext) extends HasUpdate { - final val counter = Var(0) + final val counter = Var(0) protected final val value = Var(initial) counter.trigger { @@ -27,4 +26,4 @@ class RxWithUpdate[T](initial: T, getValue: T ⇒ Future[T]) def update(): Unit = { counter() = counter.now + 1 } -} \ No newline at end of file +} diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/utils/StorageBind.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/utils/StorageBind.scala index f372f690..fdf3cc8d 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/utils/StorageBind.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/utils/StorageBind.scala @@ -8,7 +8,7 @@ trait StorageBind { def apply[T](name: String, default: T)(implicit ctx: Ctx.Owner, toString: T ⇒ String, fromString: String ⇒ T): Var[T] = { val initialValue = Option(storage.getItem(name)) - val value = Var[T](initialValue.fold(default)(fromString)) + val value = Var[T](initialValue.fold(default)(fromString)) value.triggerLater(storage.setItem(name, value.now)) value } diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/utils/TimeFormat.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/utils/TimeFormat.scala index caddbad8..6a656ade 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/utils/TimeFormat.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/utils/TimeFormat.scala @@ -36,7 +36,8 @@ private[utils] class DefaultTimeFormat(locale: String) extends TimeFormat { .toFormatter(Locale.forLanguageTag(locale)) def timestamp(ts: Long) = { - ZonedDateTime.ofInstant(Instant.ofEpochMilli(ts), ZoneId.systemDefault) + ZonedDateTime + .ofInstant(Instant.ofEpochMilli(ts), ZoneId.systemDefault) .format(timestampFormat) } } diff --git a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/utils/UploadUtils.scala b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/utils/UploadUtils.scala index 319124e6..f37bd74c 100644 --- a/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/utils/UploadUtils.scala +++ b/server/webapp/src/main/scala/com/karasiq/shadowcloud/webapp/utils/UploadUtils.scala @@ -14,27 +14,43 @@ import com.karasiq.shadowcloud.utils.Utils object UploadUtils { type XHR = dom.XMLHttpRequest - def uploadWithProgress(url: String, data: InputData, method: String = "POST", timeout: Int = 0, - headers: Map[String, String] = Map.empty, withCredentials: Boolean = false, - responseType: String = "arraybuffer"): (Rx[Int], Future[ByteString]) = { + def uploadWithProgress( + url: String, + data: InputData, + method: String = "POST", + timeout: Int = 0, + headers: Map[String, String] = Map.empty, + withCredentials: Boolean = false, + responseType: String = "arraybuffer" + ): (Rx[Int], Future[ByteString]) = { val (request, completeFuture) = doRequest(method, url, data, timeout, headers, withCredentials, responseType) - val progressRx = Var(0) + val progressRx = Var(0) - request.upload.addEventListener("progress", { (e: dom.ProgressEvent) ⇒ - // println(e.loaded + " " + e.total) - val progress = Utils.percents(e.loaded, e.total) - progressRx() = progress.toInt - }, false) + request.upload.addEventListener( + "progress", + { (e: dom.ProgressEvent) ⇒ + // println(e.loaded + " " + e.total) + val progress = Utils.percents(e.loaded, e.total) + progressRx() = progress.toInt + }, + false + ) // completeFuture.onComplete(_ ⇒ progressRx.kill()) (progressRx, completeFuture.reportFailure.responseBytes) } - private[this] def doRequest(method: String, url: String, data: InputData, timeout: Int, - headers: Map[String, String], withCredentials: Boolean, - responseType: String): (XHR, Future[XHR]) = { - val xhr = new XHR() + private[this] def doRequest( + method: String, + url: String, + data: InputData, + timeout: Int, + headers: Map[String, String], + withCredentials: Boolean, + responseType: String + ): (XHR, Future[XHR]) = { + val xhr = new XHR() val promise = Promise[XHR]() xhr.onreadystatechange = { (_: dom.Event) ⇒ diff --git a/server/webzinc-routes/src/main/scala/com/karasiq/shadowcloud/server/http/webzinc/SCAkkaHttpWebZincRoutes.scala b/server/webzinc-routes/src/main/scala/com/karasiq/shadowcloud/server/http/webzinc/SCAkkaHttpWebZincRoutes.scala index 4f86d5f1..f28b8125 100644 --- a/server/webzinc-routes/src/main/scala/com/karasiq/shadowcloud/server/http/webzinc/SCAkkaHttpWebZincRoutes.scala +++ b/server/webzinc-routes/src/main/scala/com/karasiq/shadowcloud/server/http/webzinc/SCAkkaHttpWebZincRoutes.scala @@ -37,8 +37,8 @@ trait SCAkkaHttpWebZincRoutes { self: SCAkkaHttpFileRoutes with SCAkkaHttpApiRou private[this] implicit val config = WebZincConfig(sc.config.rootConfig.getConfigIfExists("webzinc")) private[this] implicit val client = WebClient() - private[this] val fetcher = HtmlUnitWebResourceFetcher() // TODO: https://github.com/akka/akka-http/issues/86 - private[this] val inliner = WebResourceInliner() + private[this] val fetcher = HtmlUnitWebResourceFetcher() // TODO: https://github.com/akka/akka-http/issues/86 + private[this] val inliner = WebResourceInliner() def fetchWebPage(url: String): PageFuture = { val pageFuture = fetcher.getWebPage(url).flatMap((inliner.inline _).tupled) @@ -47,7 +47,8 @@ trait SCAkkaHttpWebZincRoutes { self: SCAkkaHttpFileRoutes with SCAkkaHttpApiRou def fetchHttpFile(url: String): PageFuture = { client.doHttpRequest(url).map { response ⇒ - val fileName = response.header[`Content-Disposition`] + val fileName = response + .header[`Content-Disposition`] .flatMap(_.params.get("filename")) .orElse(new URI(url).getPath.split('/').lastOption) .map(URLDecoder.decode(_, "UTF-8")) diff --git a/shell/src/main/scala/com/karasiq/shadowcloud/metadata/MetadataParserApp.scala b/shell/src/main/scala/com/karasiq/shadowcloud/metadata/MetadataParserApp.scala index 90d7dec0..7752f78d 100644 --- a/shell/src/main/scala/com/karasiq/shadowcloud/metadata/MetadataParserApp.scala +++ b/shell/src/main/scala/com/karasiq/shadowcloud/metadata/MetadataParserApp.scala @@ -13,19 +13,21 @@ import com.karasiq.shadowcloud.ShadowCloud object MetadataParserApp extends App { val actorSystem = ActorSystem("metadata-parser") - val sc = ShadowCloud(actorSystem) - val parser = sc.modules.metadata + val sc = ShadowCloud(actorSystem) + val parser = sc.modules.metadata import sc.implicits._ if (args.isEmpty) sys.error("No files specified.") Source(args.toVector) .flatMapConcat { file ⇒ - val mime = parser.getMimeType(file, ByteString(FileUtils.readFileToByteArray(new File(file)))) + val mime = parser + .getMimeType(file, ByteString(FileUtils.readFileToByteArray(new File(file)))) .getOrElse(MimeDetector.DefaultMime) println(s"File: $file") println(s"Mime type: $mime") - FileIO.fromPath(Paths.get(file)) + FileIO + .fromPath(Paths.get(file)) .via(parser.parseMetadata(file, mime)) .alsoTo(Sink.foreach { value ⇒ val (extension, bytes) = value.value match { @@ -42,7 +44,7 @@ object MetadataParserApp extends App { (thumbnail.format, thumbnail.data) case Metadata.Value.Table(table) ⇒ - ("table.txt", ByteString(table.values.map { case (key, vs) ⇒ s"$key: ${vs.values.mkString(", ")}"}.mkString("\n"))) + ("table.txt", ByteString(table.values.map { case (key, vs) ⇒ s"$key: ${vs.values.mkString(", ")}" }.mkString("\n"))) case Metadata.Value.EmbeddedResources(resources) ⇒ ("embedded.txt", ByteString(resources.toString)) @@ -50,8 +52,8 @@ object MetadataParserApp extends App { case unknown ⇒ (s"${unknown.getClass.getSimpleName}.txt", ByteString(value.toString)) } - val fileName = FilenameUtils.getName(file) + "_" + value.tag.fold("unk")(t ⇒ - s"${t.plugin}_${t.parser}_${t.disposition.toString().toLowerCase}.$extension") + val fileName = FilenameUtils.getName(file) + "_" + value.tag + .fold("unk")(t ⇒ s"${t.plugin}_${t.parser}_${t.disposition.toString().toLowerCase}.$extension") FileUtils.writeByteArrayToFile(new File(fileName), bytes.toArray) println(s"Metadata saved: $fileName") }) diff --git a/shell/src/main/scala/com/karasiq/shadowcloud/shell/ImplicitConversions.scala b/shell/src/main/scala/com/karasiq/shadowcloud/shell/ImplicitConversions.scala index 1c5d7a0b..9ec4cf19 100644 --- a/shell/src/main/scala/com/karasiq/shadowcloud/shell/ImplicitConversions.scala +++ b/shell/src/main/scala/com/karasiq/shadowcloud/shell/ImplicitConversions.scala @@ -1,6 +1,6 @@ package com.karasiq.shadowcloud.shell -import java.nio.file.{Paths, Path => FSPath} +import java.nio.file.{Paths, Path ⇒ FSPath} trait ImplicitConversions { implicit def toFSPath(path: String): FSPath = { diff --git a/shell/src/main/scala/com/karasiq/shadowcloud/shell/RegionContext.scala b/shell/src/main/scala/com/karasiq/shadowcloud/shell/RegionContext.scala index 7265c6a1..90f6cd0d 100644 --- a/shell/src/main/scala/com/karasiq/shadowcloud/shell/RegionContext.scala +++ b/shell/src/main/scala/com/karasiq/shadowcloud/shell/RegionContext.scala @@ -1,6 +1,6 @@ package com.karasiq.shadowcloud.shell -import java.nio.file.{OpenOption, StandardOpenOption, Path => FSPath} +import java.nio.file.{OpenOption, StandardOpenOption, Path ⇒ FSPath} import akka.stream.IOResult import akka.stream.scaladsl.{FileIO, Sink} @@ -59,7 +59,8 @@ private[shell] final class RegionContext(val regionId: RegionId)(implicit contex } def upload(localPath: FSPath, path: Path): File = { - val future = FileIO.fromPath(localPath) + val future = FileIO + .fromPath(localPath) .via(sc.streams.metadata.writeFileAndMetadata(regionId, path)) .map(_._1) .runWith(Sink.head) @@ -70,7 +71,7 @@ private[shell] final class RegionContext(val regionId: RegionId)(implicit contex def download(localPath: FSPath, path: Path): IOResult = { val options = Set[OpenOption](StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING) - val future = sc.streams.file.readMostRecent(regionId, path).runWith(FileIO.toPath(localPath, options)) + val future = sc.streams.file.readMostRecent(regionId, path).runWith(FileIO.toPath(localPath, options)) // ShellUtils.printIOResult(future) await(future) } diff --git a/shell/src/main/scala/com/karasiq/shadowcloud/shell/Shell.scala b/shell/src/main/scala/com/karasiq/shadowcloud/shell/Shell.scala index 14cfac21..70d3dc28 100644 --- a/shell/src/main/scala/com/karasiq/shadowcloud/shell/Shell.scala +++ b/shell/src/main/scala/com/karasiq/shadowcloud/shell/Shell.scala @@ -28,7 +28,7 @@ object Shell extends ImplicitConversions { val state = Await.result(sc.ops.supervisor.getSnapshot(), Duration.Inf) if (!state.regions.contains("test") && !state.storages.contains("test")) { - val testRegion = createRegion("test") + val testRegion = createRegion("test") val testStorage = createStorage("test", StorageProps.fromDirectory(testDirectory)) testRegion.register(testStorage) } @@ -61,7 +61,7 @@ object Shell extends ImplicitConversions { } def test(): Unit = { - val testRegion = openRegion("test") + val testRegion = openRegion("test") val testStorage = openStorage("test") testStorage.sync("test") diff --git a/shell/src/main/scala/com/karasiq/shadowcloud/shell/ShellContext.scala b/shell/src/main/scala/com/karasiq/shadowcloud/shell/ShellContext.scala index 92361f90..e3167390 100644 --- a/shell/src/main/scala/com/karasiq/shadowcloud/shell/ShellContext.scala +++ b/shell/src/main/scala/com/karasiq/shadowcloud/shell/ShellContext.scala @@ -1,7 +1,5 @@ package com.karasiq.shadowcloud.shell - - import akka.actor.ActorSystem import com.karasiq.shadowcloud.ShadowCloud @@ -12,9 +10,9 @@ private[shell] object ShellContext { } private[shell] final class ShellContext { - implicit val actorSystem = ActorSystem("shadowcloud-shell") - val sc = ShadowCloud(actorSystem) - implicit val defaultTimeout = sc.implicits.defaultTimeout - implicit val materializer = sc.implicits.materializer + implicit val actorSystem = ActorSystem("shadowcloud-shell") + val sc = ShadowCloud(actorSystem) + implicit val defaultTimeout = sc.implicits.defaultTimeout + implicit val materializer = sc.implicits.materializer implicit val executionContext = sc.implicits.executionContext } diff --git a/shell/src/main/scala/com/karasiq/shadowcloud/shell/ShellUtils.scala b/shell/src/main/scala/com/karasiq/shadowcloud/shell/ShellUtils.scala index 7194b6fd..9230e696 100644 --- a/shell/src/main/scala/com/karasiq/shadowcloud/shell/ShellUtils.scala +++ b/shell/src/main/scala/com/karasiq/shadowcloud/shell/ShellUtils.scala @@ -15,7 +15,7 @@ private[shell] object ShellUtils { def toStrings(folder: Folder): Seq[String] = { Seq("Folders:") ++ folder.folders.toSeq.sorted.map(" - " + _) ++ - Seq("Files:") ++ folder.files.toSeq.sortBy(f ⇒ (f.path.name, f.timestamp.lastModified)).map(f ⇒ s" - ${ShellUtils.toString(f)}") + Seq("Files:") ++ folder.files.toSeq.sortBy(f ⇒ (f.path.name, f.timestamp.lastModified)).map(f ⇒ s" - ${ShellUtils.toString(f)}") } def print[T](future: Future[T])(toStrings: T ⇒ GenTraversableOnce[String])(implicit ec: ExecutionContext): Unit = { diff --git a/shell/src/main/scala/com/karasiq/shadowcloud/test/Benchmark.scala b/shell/src/main/scala/com/karasiq/shadowcloud/test/Benchmark.scala index 89486302..569f1bb9 100644 --- a/shell/src/main/scala/com/karasiq/shadowcloud/test/Benchmark.scala +++ b/shell/src/main/scala/com/karasiq/shadowcloud/test/Benchmark.scala @@ -16,8 +16,8 @@ import scala.util.{Failure, Random, Success} private object Benchmark extends App { implicit val actorSystem = ActorSystem("shadowcloud-benchmark") - implicit val timeout = Timeout(15 seconds) - val sc = ShadowCloud(actorSystem) + implicit val timeout = Timeout(15 seconds) + val sc = ShadowCloud(actorSystem) import sc.implicits._ printBlock("Plugins") @@ -48,24 +48,26 @@ private object Benchmark extends App { // Benchmarks private[this] def runProviderBenchmark(provider: String, encAlg: String, encKeySize: Int, hashAlg: String): Unit = { printBlock(s"$provider $encAlg[$encKeySize]/$hashAlg benchmark") - val encMethod = EncryptionMethod(encAlg, encKeySize, provider = provider) + val encMethod = EncryptionMethod(encAlg, encKeySize, provider = provider) val hashMethod = HashingMethod(hashAlg, provider = provider) runWriteBenchmark(encMethod, hashMethod, HashingMethod.none) runWriteBenchmark(encMethod, hashMethod, hashMethod) // Double hashing } - private[this] def runWriteBenchmark(encryption: EncryptionMethod = sc.config.crypto.encryption.chunks, - hashing: HashingMethod = sc.config.crypto.hashing.chunks, - fileHashing: HashingMethod = sc.config.crypto.hashing.files): Unit = { - val modifier = 1 - val chunkSize = SizeUnit.MB.intValue * modifier + private[this] def runWriteBenchmark( + encryption: EncryptionMethod = sc.config.crypto.encryption.chunks, + hashing: HashingMethod = sc.config.crypto.hashing.chunks, + fileHashing: HashingMethod = sc.config.crypto.hashing.files + ): Unit = { + val modifier = 1 + val chunkSize = SizeUnit.MB.intValue * modifier val chunkCount = 1024 / modifier - val mbCount = chunkCount * (chunkSize.toDouble / SizeUnit.MB) + val mbCount = chunkCount * (chunkSize.toDouble / SizeUnit.MB) println(s"Starting write benchmark (${MemorySize(chunkSize)}): $encryption/$hashing/$fileHashing") try { val startTime = System.nanoTime() - val promise = Promise[Done] + val promise = Promise[Done] randomBytesSource(chunkSize) .via(ChunkSplitter(chunkSize)) .take(chunkCount) @@ -76,8 +78,8 @@ private object Benchmark extends App { .runWith(Sink.onComplete { case Success(Done) ⇒ val elapsed = (System.nanoTime() - startTime).nanos - val perMb = elapsed / mbCount - val speed = 1.second / perMb + val perMb = elapsed / mbCount + val speed = 1.second / perMb println(f"Write benchmark completed, ${elapsed.toSeconds} seconds elapsed, ${perMb.toMillis} ms per megabyte, $speed%.2f MB/sec") promise.success(Done) @@ -92,17 +94,20 @@ private object Benchmark extends App { private[this] def runReadBenchmark(): Unit = { val start = System.nanoTime() - val future = Source.future(sc.ops.region.getFolderIndex("testRegion")) + val future = Source + .future(sc.ops.region.getFolderIndex("testRegion")) .map(_.filesIterator.maxBy(_.checksum.size)) .flatMapConcat(file ⇒ sc.streams.file.read("testRegion", file)) .map(_.length) .runWith(Sink.fold(0L)(_ + _)) - val bytes = Await.result(future, Duration.Inf) + val bytes = Await.result(future, Duration.Inf) val elapsed = (System.nanoTime() - start).nanos - val perMb = elapsed / (bytes / SizeUnit.MB) - val speed = 1.second / perMb - println(f"Read benchmark completed, ${MemorySize(bytes)}, ${elapsed.toSeconds} seconds elapsed, ${perMb.toMillis} ms per megabyte, $speed%.2f MB/sec") + val perMb = elapsed / (bytes / SizeUnit.MB) + val speed = 1.second / perMb + println( + f"Read benchmark completed, ${MemorySize(bytes)}, ${elapsed.toSeconds} seconds elapsed, ${perMb.toMillis} ms per megabyte, $speed%.2f MB/sec" + ) } // Utils diff --git a/storage/dropbox/src/main/scala/com/karasiq/shadowcloud/dropbox/DropboxHealthProvider.scala b/storage/dropbox/src/main/scala/com/karasiq/shadowcloud/dropbox/DropboxHealthProvider.scala index 6aa98cac..c34267e0 100644 --- a/storage/dropbox/src/main/scala/com/karasiq/shadowcloud/dropbox/DropboxHealthProvider.scala +++ b/storage/dropbox/src/main/scala/com/karasiq/shadowcloud/dropbox/DropboxHealthProvider.scala @@ -14,7 +14,7 @@ object DropboxHealthProvider { class DropboxHealthProvider(dropboxClient: DropboxClient)(implicit ec: ExecutionContext) extends StorageHealthProvider { def health = dropboxClient.spaceUsage().map { spaceUsage ⇒ - val used = spaceUsage.getUsed + val used = spaceUsage.getUsed val total = spaceUsage.getAllocation.getIndividualValue.getAllocated StorageHealth.normalized(total - used, total, used) } diff --git a/storage/dropbox/src/main/scala/com/karasiq/shadowcloud/dropbox/DropboxRepository.scala b/storage/dropbox/src/main/scala/com/karasiq/shadowcloud/dropbox/DropboxRepository.scala index 342a62e6..d4e684e3 100644 --- a/storage/dropbox/src/main/scala/com/karasiq/shadowcloud/dropbox/DropboxRepository.scala +++ b/storage/dropbox/src/main/scala/com/karasiq/shadowcloud/dropbox/DropboxRepository.scala @@ -21,7 +21,8 @@ object DropboxRepository { class DropboxRepository(dropboxClient: DropboxClient)(implicit ec: ExecutionContext) extends PathTreeRepository { def subKeys(fromPath: Path) = { require(Path.isConventional(fromPath), s"Non-conventional: $fromPath") - dropboxClient.list(fromPath.toString, recursive = true) + dropboxClient + .list(fromPath.toString, recursive = true) .map(metadata ⇒ Path.fromString(metadata.getPathDisplay)) .alsoToMat(StorageUtils.countPassedElements(fromPath).toMat(Sink.head)(Keep.right))(Keep.right) .recoverWithRetries(1, { case e: ListFolderErrorException if e.errorValue.getPathValue.isNotFound ⇒ Source.empty }) @@ -30,7 +31,8 @@ class DropboxRepository(dropboxClient: DropboxClient)(implicit ec: ExecutionCont def read(path: Path) = { require(Path.isConventional(path), s"Non-conventional: $path") - dropboxClient.download(path.toString) + dropboxClient + .download(path.toString) .alsoToMat(StorageUtils.countPassedBytes(path).toMat(Sink.head)(Keep.right))(Keep.right) } diff --git a/storage/dropbox/src/main/scala/com/karasiq/shadowcloud/dropbox/DropboxSessionProxy.scala b/storage/dropbox/src/main/scala/com/karasiq/shadowcloud/dropbox/DropboxSessionProxy.scala index 0ca71ac3..92b464de 100644 --- a/storage/dropbox/src/main/scala/com/karasiq/shadowcloud/dropbox/DropboxSessionProxy.scala +++ b/storage/dropbox/src/main/scala/com/karasiq/shadowcloud/dropbox/DropboxSessionProxy.scala @@ -38,49 +38,47 @@ object DropboxSessionProxy { val appKeys = Dropbox.AppKeys(config.getConfigIfExists("app-keys")) - SessionProxyActor.props( - implicit context ⇒ - new SessionAuthenticator[UserToken] { - import context.{dispatcher, system} - import sc.implicits.materializer + SessionProxyActor.props(implicit context ⇒ + new SessionAuthenticator[UserToken] { + import context.{dispatcher, system} + import sc.implicits.materializer - def getSession(): Future[UserToken] = { - def checkConnection(): Future[Done] = - Http() - .singleRequest(HttpRequest(uri = "https://dropbox.com/")) - .filter(_.status.isSuccess()) - .recover { case _ ⇒ throw new IllegalStateException("No connection") } - .map(_ ⇒ Done) + def getSession(): Future[UserToken] = { + def checkConnection(): Future[Done] = + Http() + .singleRequest(HttpRequest(uri = "https://dropbox.com/")) + .filter(_.status.isSuccess()) + .recover { case _ ⇒ throw new IllegalStateException("No connection") } + .map(_ ⇒ Done) - val cachedToken = for { - _token ← sc.sessions.get[UserToken](storageId, "oauth") - _ ← { implicit val token = _token; DropboxClient(SCDropbox.DispatcherId).spaceUsage() } - } yield _token + val cachedToken = for { + _token ← sc.sessions.get[UserToken](storageId, "oauth") + _ ← { implicit val token = _token; DropboxClient(SCDropbox.DispatcherId).spaceUsage() } + } yield _token - cachedToken.recoverWith { - case _: NoSuchElementException ⇒ - val oauth = DropboxOAuth() - for { - _ ← checkConnection() - _ = sc.ui.showNotification( - s"Please authorize shadowcloud in your Dropbox acc: $storageId (${props.credentials.login})\nPress OK to open OAuth web page" - ) - token ← oauth.authenticate(appKeys) - Done ← sc.sessions.set(storageId, "oauth", token) - } yield token - } + cachedToken.recoverWith { case _: NoSuchElementException ⇒ + val oauth = DropboxOAuth() + for { + _ ← checkConnection() + _ = sc.ui.showNotification( + s"Please authorize shadowcloud in your Dropbox acc: $storageId (${props.credentials.login})\nPress OK to open OAuth web page" + ) + token ← oauth.authenticate(appKeys) + Done ← sc.sessions.set(storageId, "oauth", token) + } yield token } + } - def createDispatcher(_token: UserToken): ActorRef = { - implicit val token = _token - val client = DropboxClient(SCDropbox.DispatcherId) - StoragePluginBuilder(storageId, props) - .withIndexTree(DropboxRepository(client)) - .withChunksTree(DropboxRepository(client)) - .withHealth(DropboxHealthProvider(client)(context.dispatcher)) - .createStorage() - } + def createDispatcher(_token: UserToken): ActorRef = { + implicit val token = _token + val client = DropboxClient(SCDropbox.DispatcherId) + StoragePluginBuilder(storageId, props) + .withIndexTree(DropboxRepository(client)) + .withChunksTree(DropboxRepository(client)) + .withHealth(DropboxHealthProvider(client)(context.dispatcher)) + .createStorage() } + } ) } } diff --git a/storage/dropbox/src/main/scala/com/karasiq/shadowcloud/dropbox/DropboxStorageProvider.scala b/storage/dropbox/src/main/scala/com/karasiq/shadowcloud/dropbox/DropboxStorageProvider.scala index 4ea4dc75..13ba8fc7 100644 --- a/storage/dropbox/src/main/scala/com/karasiq/shadowcloud/dropbox/DropboxStorageProvider.scala +++ b/storage/dropbox/src/main/scala/com/karasiq/shadowcloud/dropbox/DropboxStorageProvider.scala @@ -11,8 +11,7 @@ class DropboxStorageProvider extends StorageProvider { DropboxStoragePlugin() } - override val storageConfigs: StorageConfigPF = { - case "dropbox" ⇒ - ConfigProps("type" → "dropbox", "credentials.login" → "example@mail.com") + override val storageConfigs: StorageConfigPF = { case "dropbox" ⇒ + ConfigProps("type" → "dropbox", "credentials.login" → "example@mail.com") } -} \ No newline at end of file +} diff --git a/storage/gdrive/src/main/scala/com/karasiq/shadowcloud/storage/gdrive/GDriveDispatchers.scala b/storage/gdrive/src/main/scala/com/karasiq/shadowcloud/storage/gdrive/GDriveDispatchers.scala index 35ea108e..e8c8c403 100644 --- a/storage/gdrive/src/main/scala/com/karasiq/shadowcloud/storage/gdrive/GDriveDispatchers.scala +++ b/storage/gdrive/src/main/scala/com/karasiq/shadowcloud/storage/gdrive/GDriveDispatchers.scala @@ -1,6 +1,6 @@ package com.karasiq.shadowcloud.storage.gdrive private[gdrive] object GDriveDispatchers { - def apiDispatcherId = "shadowcloud.storage.gdrive.api-dispatcher" + def apiDispatcherId = "shadowcloud.storage.gdrive.api-dispatcher" def fileDispatcherId = "shadowcloud.storage.gdrive.file-dispatcher" } diff --git a/storage/gdrive/src/main/scala/com/karasiq/shadowcloud/storage/gdrive/GDriveHealthProvider.scala b/storage/gdrive/src/main/scala/com/karasiq/shadowcloud/storage/gdrive/GDriveHealthProvider.scala index ecaefd0c..3eb03550 100644 --- a/storage/gdrive/src/main/scala/com/karasiq/shadowcloud/storage/gdrive/GDriveHealthProvider.scala +++ b/storage/gdrive/src/main/scala/com/karasiq/shadowcloud/storage/gdrive/GDriveHealthProvider.scala @@ -15,7 +15,8 @@ private[gdrive] object GDriveHealthProvider { } } -private[gdrive] class GDriveHealthProvider(service: GDriveService, props: StorageProps)(implicit ec: ExecutionContext, td: TeamDriveId) extends StorageHealthProvider { +private[gdrive] class GDriveHealthProvider(service: GDriveService, props: StorageProps)(implicit ec: ExecutionContext, td: TeamDriveId) + extends StorageHealthProvider { lazy val estimator = GDriveSpaceEstimator(service, StoragePluginBuilder.getRootPath(props)) def health = { diff --git a/storage/gdrive/src/main/scala/com/karasiq/shadowcloud/storage/gdrive/GDriveRepository.scala b/storage/gdrive/src/main/scala/com/karasiq/shadowcloud/storage/gdrive/GDriveRepository.scala index 007f318e..6d67f629 100644 --- a/storage/gdrive/src/main/scala/com/karasiq/shadowcloud/storage/gdrive/GDriveRepository.scala +++ b/storage/gdrive/src/main/scala/com/karasiq/shadowcloud/storage/gdrive/GDriveRepository.scala @@ -36,10 +36,9 @@ private[gdrive] class GDriveRepository(service: GDriveService)(implicit ec: Exec def iteratorTry(path: Path) = Try { service.traverseFolder(path.nodes) - } recover { - case _ ⇒ - Await.result(entityCache.getOrCreateFolderId(path), 3 minutes) - service.traverseFolder(path.nodes) + } recover { case _ ⇒ + Await.result(entityCache.getOrCreateFolderId(path), 3 minutes) + service.traverseFolder(path.nodes) } Flow[Path] @@ -59,10 +58,9 @@ private[gdrive] class GDriveRepository(service: GDriveService)(implicit ec: Exec .toMat(Sink.head)(Keep.right) )(Keep.right) // .log("gdrive-entities") - .map { - case (pathNodes, file) ⇒ - val parent = Path(pathNodes).toRelative(fromPath) - parent / file.name + .map { case (pathNodes, file) ⇒ + val parent = Path(pathNodes).toRelative(fromPath) + parent / file.name } .named("gdriveKeys") } @@ -107,18 +105,17 @@ private[gdrive] class GDriveRepository(service: GDriveService)(implicit ec: Exec Flow[Data] .via(AkkaStreamUtils.extractUpstream) .zip(Source.lazyFuture(() ⇒ getFolderId(path))) - .flatMapConcat { - case (stream, folderId) ⇒ - stream.via( - AkkaStreamUtils.writeInputStream( - { inputStream ⇒ - val result = Try(blockingUpload(folderId, path.name, inputStream)) - result.foreach(_ ⇒ entityCache.resetFileCache(path)) - Source.future(Future.fromTry(result)) - }, - Dispatcher(GDriveDispatchers.fileDispatcherId) - ) + .flatMapConcat { case (stream, folderId) ⇒ + stream.via( + AkkaStreamUtils.writeInputStream( + { inputStream ⇒ + val result = Try(blockingUpload(folderId, path.name, inputStream)) + result.foreach(_ ⇒ entityCache.resetFileCache(path)) + Source.future(Future.fromTry(result)) + }, + Dispatcher(GDriveDispatchers.fileDispatcherId) ) + ) } .map(written ⇒ StorageIOResult.Success(path, written)) .withAttributes(fileStreamAttributes) diff --git a/storage/gdrive/src/main/scala/com/karasiq/shadowcloud/storage/gdrive/GDriveSpaceEstimator.scala b/storage/gdrive/src/main/scala/com/karasiq/shadowcloud/storage/gdrive/GDriveSpaceEstimator.scala index 18e1735b..9493da72 100644 --- a/storage/gdrive/src/main/scala/com/karasiq/shadowcloud/storage/gdrive/GDriveSpaceEstimator.scala +++ b/storage/gdrive/src/main/scala/com/karasiq/shadowcloud/storage/gdrive/GDriveSpaceEstimator.scala @@ -31,10 +31,9 @@ private[gdrive] class GDriveSpaceEstimator(drive: GDriveService, rootPath: Path) ec.execute { () ⇒ var size = 0L - Try(drive.traverseFolder(rootPath.nodes).foreach { - case (_, file) ⇒ - size += file.size - if (size > lastEstimated) lastEstimated = size + Try(drive.traverseFolder(rootPath.nodes).foreach { case (_, file) ⇒ + size += file.size + if (size > lastEstimated) lastEstimated = size }) lastEstimated = size } diff --git a/storage/gdrive/src/main/scala/com/karasiq/shadowcloud/storage/gdrive/GDriveStoragePlugin.scala b/storage/gdrive/src/main/scala/com/karasiq/shadowcloud/storage/gdrive/GDriveStoragePlugin.scala index c23f605b..fbf15260 100644 --- a/storage/gdrive/src/main/scala/com/karasiq/shadowcloud/storage/gdrive/GDriveStoragePlugin.scala +++ b/storage/gdrive/src/main/scala/com/karasiq/shadowcloud/storage/gdrive/GDriveStoragePlugin.scala @@ -29,7 +29,7 @@ private[gdrive] class GDriveStoragePlugin(implicit sc: ShadowCloudExtension) ext .withFallback(defaultConfig) val proxyProps = Props(new Actor { - import context.{dispatcher => executionContext} // API dispatcher + import context.{dispatcher ⇒ executionContext} // API dispatcher def receiveAuthorized(storageDispatcher: ActorRef): Receive = { case message if sender() == storageDispatcher ⇒ @@ -39,50 +39,49 @@ private[gdrive] class GDriveStoragePlugin(implicit sc: ShadowCloudExtension) ext storageDispatcher.forward(message) } - def receive = { - case message ⇒ - implicit val driveContext = { - val dataStore = SCGDriveStore(storageId, props.credentials.login) - GDriveContext(config, dataStore) - } - - val oauth = GDriveOAuth() - - val sessions = sc.sessions.listBlocking(storageId) - if (sessions.isEmpty) - sc.ui.showNotification( - s"Please authorize shadowcloud in your Google Drive acc: $storageId (${props.credentials.login})\nPress OK to open OAuth web page" - ) - - implicit val session = oauth.authorize(props.credentials.login) - - val service = { - val applicationName = config.withDefault("shadowcloud", _.getString("application-name")) - GDriveService(applicationName) - } - - implicit val teamDriveId: TeamDriveId = { - val driveId = props.rootConfig - .optional(_.getString("team-drive")) - .filter(_.nonEmpty) - .map { name => - val drives = service.teamDrives() - drives - .find(_.getName == name) - .map(_.getId) - .getOrElse(throw new IllegalArgumentException(s"Team drive not found: $name, available: $drives")) - } - TeamDriveId(driveId.orNull) - } - - val dispatcher = StoragePluginBuilder(storageId, props) - .withIndexTree(GDriveRepository(service)) - .withChunksTree(GDriveRepository(service)) - .withHealth(GDriveHealthProvider(service, props)) - .createStorage() - - context.become(receiveAuthorized(dispatcher)) - self.forward(message) + def receive = { case message ⇒ + implicit val driveContext = { + val dataStore = SCGDriveStore(storageId, props.credentials.login) + GDriveContext(config, dataStore) + } + + val oauth = GDriveOAuth() + + val sessions = sc.sessions.listBlocking(storageId) + if (sessions.isEmpty) + sc.ui.showNotification( + s"Please authorize shadowcloud in your Google Drive acc: $storageId (${props.credentials.login})\nPress OK to open OAuth web page" + ) + + implicit val session = oauth.authorize(props.credentials.login) + + val service = { + val applicationName = config.withDefault("shadowcloud", _.getString("application-name")) + GDriveService(applicationName) + } + + implicit val teamDriveId: TeamDriveId = { + val driveId = props.rootConfig + .optional(_.getString("team-drive")) + .filter(_.nonEmpty) + .map { name ⇒ + val drives = service.teamDrives() + drives + .find(_.getName == name) + .map(_.getId) + .getOrElse(throw new IllegalArgumentException(s"Team drive not found: $name, available: $drives")) + } + TeamDriveId(driveId.orNull) + } + + val dispatcher = StoragePluginBuilder(storageId, props) + .withIndexTree(GDriveRepository(service)) + .withChunksTree(GDriveRepository(service)) + .withHealth(GDriveHealthProvider(service, props)) + .createStorage() + + context.become(receiveAuthorized(dispatcher)) + self.forward(message) } }) diff --git a/storage/gdrive/src/main/scala/com/karasiq/shadowcloud/storage/gdrive/GDriveStorageProvider.scala b/storage/gdrive/src/main/scala/com/karasiq/shadowcloud/storage/gdrive/GDriveStorageProvider.scala index 1a7c9508..368a2471 100644 --- a/storage/gdrive/src/main/scala/com/karasiq/shadowcloud/storage/gdrive/GDriveStorageProvider.scala +++ b/storage/gdrive/src/main/scala/com/karasiq/shadowcloud/storage/gdrive/GDriveStorageProvider.scala @@ -12,8 +12,7 @@ class GDriveStorageProvider(sc: ShadowCloudExtension) extends StorageProvider { GDriveStoragePlugin(sc) } - override def storageConfigs: StorageConfigPF = { - case "gdrive" ⇒ - ConfigProps("type" → "gdrive", "credentials.login" → "example@gmail.com", "team-drive" -> "") + override def storageConfigs: StorageConfigPF = { case "gdrive" ⇒ + ConfigProps("type" → "gdrive", "credentials.login" → "example@gmail.com", "team-drive" → "") } } diff --git a/storage/gdrive/src/main/scala/com/karasiq/shadowcloud/storage/gdrive/SCGDriveStore.scala b/storage/gdrive/src/main/scala/com/karasiq/shadowcloud/storage/gdrive/SCGDriveStore.scala index 2aa78a93..ad0e6ad7 100644 --- a/storage/gdrive/src/main/scala/com/karasiq/shadowcloud/storage/gdrive/SCGDriveStore.scala +++ b/storage/gdrive/src/main/scala/com/karasiq/shadowcloud/storage/gdrive/SCGDriveStore.scala @@ -24,7 +24,7 @@ class SCGDriveStore(storageId: StorageId, userId: String)(implicit sc: ShadowClo } private final class SCStore[V <: AnyRef with Serializable](id: String) extends AbstractDataStore[V](SCGDriveStore.this, id) { - private[this] val prefix = "gdrive_" + userId + "_" + id + "_" + private[this] val prefix = "gdrive_" + userId + "_" + id + "_" private[this] val timeout = 10 seconds def set(key: String, value: V): DataStore[V] = { @@ -59,7 +59,8 @@ class SCGDriveStore(storageId: StorageId, userId: String)(implicit sc: ShadowClo } def get(key: String): V = { - val future = sc.sessions.provider.loadSession(storageId, prefix + key) + val future = sc.sessions.provider + .loadSession(storageId, prefix + key) .map(deserialize) .recover { case _ ⇒ null.asInstanceOf[V] } @@ -67,7 +68,8 @@ class SCGDriveStore(storageId: StorageId, userId: String)(implicit sc: ShadowClo } private[this] def prefixedKeys() = { - sc.sessions.provider.getSessions(storageId) + sc.sessions.provider + .getSessions(storageId) .map(keys ⇒ keys.filter(_.startsWith(prefix))) } diff --git a/storage/mailrucloud/src/main/scala/com/karasiq/shadowcloud/mailrucloud/MailRuCloudHealthProvider.scala b/storage/mailrucloud/src/main/scala/com/karasiq/shadowcloud/mailrucloud/MailRuCloudHealthProvider.scala index a18df05a..42cd741a 100644 --- a/storage/mailrucloud/src/main/scala/com/karasiq/shadowcloud/mailrucloud/MailRuCloudHealthProvider.scala +++ b/storage/mailrucloud/src/main/scala/com/karasiq/shadowcloud/mailrucloud/MailRuCloudHealthProvider.scala @@ -13,7 +13,8 @@ object MailRuCloudHealthProvider { } } -class MailRuCloudHealthProvider(client: MailCloudClient)(implicit ec: ExecutionContext, session: Session, token: CsrfToken) extends StorageHealthProvider { +class MailRuCloudHealthProvider(client: MailCloudClient)(implicit ec: ExecutionContext, session: Session, token: CsrfToken) + extends StorageHealthProvider { def health = { client.space.map(spc ⇒ StorageHealth.normalized(spc.total - spc.used, spc.total, spc.used)) } diff --git a/storage/mailrucloud/src/main/scala/com/karasiq/shadowcloud/mailrucloud/MailRuCloudProxyActor.scala b/storage/mailrucloud/src/main/scala/com/karasiq/shadowcloud/mailrucloud/MailRuCloudProxyActor.scala index 4510f3c4..c984346e 100644 --- a/storage/mailrucloud/src/main/scala/com/karasiq/shadowcloud/mailrucloud/MailRuCloudProxyActor.scala +++ b/storage/mailrucloud/src/main/scala/com/karasiq/shadowcloud/mailrucloud/MailRuCloudProxyActor.scala @@ -16,40 +16,42 @@ private[mailrucloud] object MailRuCloudProxyActor { final case class MailCloudSession(session: Session, token: CsrfToken, nodes: Nodes) def props(storageId: StorageId, props: StorageProps): Props = { - SessionProxyActor.props(implicit context ⇒ new SessionAuthenticator[MailCloudSession] { - import context.dispatcher - implicit val sc = ShadowCloud() - val client = MailCloudClient()(context.system) - - override def getSession(): Future[MailCloudSession] = { - def getSessionCached() = { - for { - session ← sc.sessions.get[Session](storageId, "mailcloud-session") - token ← client.csrfToken(session) - nodes ← client.nodes(session, token) - } yield MailCloudSession(session, token, nodes) + SessionProxyActor.props(implicit context ⇒ + new SessionAuthenticator[MailCloudSession] { + import context.dispatcher + implicit val sc = ShadowCloud() + val client = MailCloudClient()(context.system) + + override def getSession(): Future[MailCloudSession] = { + def getSessionCached() = { + for { + session ← sc.sessions.get[Session](storageId, "mailcloud-session") + token ← client.csrfToken(session) + nodes ← client.nodes(session, token) + } yield MailCloudSession(session, token, nodes) + } + + def getSession() = { + for { + session ← client.login(props.credentials.login, props.credentials.password) + _ ← sc.sessions.set(storageId, "mailcloud-session", session) + token ← client.csrfToken(session) + nodes ← client.nodes(session, token) + } yield MailCloudSession(session, token, nodes) + } + + getSessionCached().recoverWith { case _ ⇒ getSession() } } - def getSession() = { - for { - session ← client.login(props.credentials.login, props.credentials.password) - _ ← sc.sessions.set(storageId, "mailcloud-session", session) - token ← client.csrfToken(session) - nodes ← client.nodes(session, token) - } yield MailCloudSession(session, token, nodes) + def createDispatcher(_session: MailCloudSession) = { + implicit val MailCloudSession(session, token, nodes) = _session + StoragePluginBuilder(storageId, props) + .withChunksTree(MailRuCloudRepository(client)) + .withIndexTree(MailRuCloudRepository(client)) + .withHealth(MailRuCloudHealthProvider(client)) + .createStorage() } - - getSessionCached().recoverWith { case _ ⇒ getSession() } - } - - def createDispatcher(_session: MailCloudSession) = { - implicit val MailCloudSession(session, token, nodes) = _session - StoragePluginBuilder(storageId, props) - .withChunksTree(MailRuCloudRepository(client)) - .withIndexTree(MailRuCloudRepository(client)) - .withHealth(MailRuCloudHealthProvider(client)) - .createStorage() } - }) + ) } } diff --git a/storage/mailrucloud/src/main/scala/com/karasiq/shadowcloud/mailrucloud/MailRuCloudRepository.scala b/storage/mailrucloud/src/main/scala/com/karasiq/shadowcloud/mailrucloud/MailRuCloudRepository.scala index 60062791..e151b1c3 100644 --- a/storage/mailrucloud/src/main/scala/com/karasiq/shadowcloud/mailrucloud/MailRuCloudRepository.scala +++ b/storage/mailrucloud/src/main/scala/com/karasiq/shadowcloud/mailrucloud/MailRuCloudRepository.scala @@ -38,14 +38,15 @@ object MailRuCloudRepository { .named("countBytes") } -class MailRuCloudRepository(client: MailCloudClient)(implicit nodes: Nodes, session: Session, - token: CsrfToken, ec: ExecutionContext) extends PathTreeRepository { +class MailRuCloudRepository(client: MailCloudClient)(implicit nodes: Nodes, session: Session, token: CsrfToken, ec: ExecutionContext) + extends PathTreeRepository { import MailRuCloudRepository._ private[this] val foldersCache = CacheMap[Path, Path] def read(path: Path) = { - client.download(path) + client + .download(path) .alsoToMat(countBytes(path))(Keep.right) .named("mailrucloudRead") } @@ -69,14 +70,18 @@ class MailRuCloudRepository(client: MailCloudClient)(implicit nodes: Nodes, sess Flow[Path] .flatMapConcat(path ⇒ Source.future(client.file(path)).map((path, _))) .withAttributes(ActorAttributes.supervisionStrategy(Supervision.resumingDecider)) - .flatMapMerge(2, { case (path, file) ⇒ - Source.future(client.delete(path)) - .map(_ ⇒ file.size) - .recover { - case ae: ApiException if ae.errorName.contains("not_exists") => 0L - } - .via(StorageUtils.wrapCountStream(path)) - }) + .flatMapMerge( + 2, + { case (path, file) ⇒ + Source + .future(client.delete(path)) + .map(_ ⇒ file.size) + .recover { + case ae: ApiException if ae.errorName.contains("not_exists") ⇒ 0L + } + .via(StorageUtils.wrapCountStream(path)) + } + ) .via(StorageUtils.foldStream()) .toMat(Sink.head)(Keep.right) .named("mailrucloudDelete") @@ -90,7 +95,8 @@ class MailRuCloudRepository(client: MailCloudClient)(implicit nodes: Nodes, sess offsets.iterator.map(client.folder(path, _, filesPerRequest)) } - Source.fromIterator(() ⇒ createFuturesIterator()) + Source + .fromIterator(() ⇒ createFuturesIterator()) .mapAsync(1)(identity) .takeWhile(_.list.nonEmpty) .orElse(Source.single(MailCloudTypes.Folder("folder", "folder", path.name, path.toString, 0, ""))) @@ -109,25 +115,29 @@ class MailRuCloudRepository(client: MailCloudClient)(implicit nodes: Nodes, sess .recoverWithRetries(1, { case ae: ApiException if ae.errorName.contains("not_exists") ⇒ Source.empty }) .named("mailrucloudTraverse") - Source.single(fromPath) + Source + .single(fromPath) .via(traverseFlow) .map(_.toRelative(fromPath)) - .alsoToMat(Flow[Path] - .fold(0L)((c, _) ⇒ c + 1) - .via(StorageUtils.wrapCountStream(fromPath)) - .toMat(Sink.head)(Keep.right) + .alsoToMat( + Flow[Path] + .fold(0L)((c, _) ⇒ c + 1) + .via(StorageUtils.wrapCountStream(fromPath)) + .toMat(Sink.head)(Keep.right) )(Keep.right) .named("mailrucloudKeys") } private[this] def getOrCreateFolder(path: Path): Source[Path, NotUsed] = { val future = foldersCache(path) { - client.createFolder(path) + client + .createFolder(path) .map(ep ⇒ ep: Path) .recover { case ae: ApiException if ae.errorName.contains("exists") ⇒ path } } - Source.future(future) + Source + .future(future) .named("mailrucloudCreateFolder") - } + } } diff --git a/storage/mailrucloud/src/main/scala/com/karasiq/shadowcloud/mailrucloud/MailRuCloudStorageProvider.scala b/storage/mailrucloud/src/main/scala/com/karasiq/shadowcloud/mailrucloud/MailRuCloudStorageProvider.scala index ebd34276..61745fed 100644 --- a/storage/mailrucloud/src/main/scala/com/karasiq/shadowcloud/mailrucloud/MailRuCloudStorageProvider.scala +++ b/storage/mailrucloud/src/main/scala/com/karasiq/shadowcloud/mailrucloud/MailRuCloudStorageProvider.scala @@ -11,8 +11,7 @@ class MailRuCloudStorageProvider extends StorageProvider { MailRuCloudStoragePlugin() } - override def storageConfigs = { - case "mailrucloud" ⇒ - ConfigProps("type" → "mailrucloud", "credentials.login" → "example@mail.ru", "credentials.password" → "123456") + override def storageConfigs = { case "mailrucloud" ⇒ + ConfigProps("type" → "mailrucloud", "credentials.login" → "example@mail.ru", "credentials.password" → "123456") } } diff --git a/storage/parent/src/main/scala/com/karasiq/shadowcloud/actors/SessionProxyActor.scala b/storage/parent/src/main/scala/com/karasiq/shadowcloud/actors/SessionProxyActor.scala index c051f303..72652548 100644 --- a/storage/parent/src/main/scala/com/karasiq/shadowcloud/actors/SessionProxyActor.scala +++ b/storage/parent/src/main/scala/com/karasiq/shadowcloud/actors/SessionProxyActor.scala @@ -16,7 +16,7 @@ private[shadowcloud] object SessionProxyActor { private[shadowcloud] class SessionProxyActor[Session](createAuthenticator: ActorContext ⇒ SessionAuthenticator[Session]) extends Actor with Stash { import context.dispatcher - val state = createAuthenticator(context) + val state = createAuthenticator(context) val timeoutSchedule = context.system.scheduler.scheduleOnce(5 minutes, self, ReceiveTimeout) def receive = { diff --git a/storage/parent/src/main/scala/com/karasiq/shadowcloud/providers/StorageProvider.scala b/storage/parent/src/main/scala/com/karasiq/shadowcloud/providers/StorageProvider.scala index 8f1ea920..2300850a 100644 --- a/storage/parent/src/main/scala/com/karasiq/shadowcloud/providers/StorageProvider.scala +++ b/storage/parent/src/main/scala/com/karasiq/shadowcloud/providers/StorageProvider.scala @@ -1,16 +1,14 @@ package com.karasiq.shadowcloud.providers - - import com.karasiq.shadowcloud.config.SerializedProps import com.karasiq.shadowcloud.storage.StoragePlugin import com.karasiq.shadowcloud.storage.props.StorageProps abstract class StorageProvider extends ModuleProvider { - type StoragePF = PartialFunction[StorageProps, StoragePlugin] + type StoragePF = PartialFunction[StorageProps, StoragePlugin] type StorageConfigPF = PartialFunction[String, SerializedProps] - def storageTypes: Set[String] = Set.empty - def storages: StoragePF = PartialFunction.empty + def storageTypes: Set[String] = Set.empty + def storages: StoragePF = PartialFunction.empty def storageConfigs: StorageConfigPF = PartialFunction.empty } diff --git a/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/StorageHealthProvider.scala b/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/StorageHealthProvider.scala index 54430d7e..9fed3064 100644 --- a/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/StorageHealthProvider.scala +++ b/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/StorageHealthProvider.scala @@ -21,10 +21,11 @@ object StorageHealthProvider { case _ ⇒ QuotedStorageHealthProvider(hp, quota) } - - private final case class QuotedStorageHealthProvider(underlying: StorageHealthProvider, quota: Quota)(implicit ec: ExecutionContext) extends StorageHealthProvider { + + private final case class QuotedStorageHealthProvider(underlying: StorageHealthProvider, quota: Quota)(implicit ec: ExecutionContext) + extends StorageHealthProvider { def health: Future[StorageHealth] = underlying.health.map { sh ⇒ - val newTotalSpace = Quota.limitTotalSpace(quota, sh.totalSpace) + val newTotalSpace = Quota.limitTotalSpace(quota, sh.totalSpace) val newWritableSpace = if (quota.readOnly) 0L else math.min(sh.writableSpace, newTotalSpace - sh.usedSpace) StorageHealth.normalized(newWritableSpace, newTotalSpace, sh.usedSpace, sh.online) } diff --git a/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/StoragePlugin.scala b/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/StoragePlugin.scala index 0c9eb39c..8aa7f4f4 100644 --- a/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/StoragePlugin.scala +++ b/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/StoragePlugin.scala @@ -1,7 +1,5 @@ package com.karasiq.shadowcloud.storage - - import akka.actor.{ActorContext, ActorRef} import com.karasiq.shadowcloud.model.StorageId import com.karasiq.shadowcloud.storage.props.StorageProps diff --git a/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/props/StorageProps.scala b/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/props/StorageProps.scala index 624a8eb6..046d117b 100644 --- a/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/props/StorageProps.scala +++ b/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/props/StorageProps.scala @@ -1,7 +1,7 @@ package com.karasiq.shadowcloud.storage.props import java.net.URI -import java.nio.file.{Path => FSPath} +import java.nio.file.{Path ⇒ FSPath} import com.karasiq.common.configs.ConfigImplicits import com.karasiq.common.memory.MemorySize @@ -12,8 +12,6 @@ import com.karasiq.shadowcloud.storage.props.StorageProps.{Address, Credentials, import com.karasiq.shadowcloud.utils.Utils import com.typesafe.config.Config - - @SerialVersionUID(0L) final case class StorageProps( rootConfig: Config, diff --git a/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/repository/CategorizedRepository.scala b/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/repository/CategorizedRepository.scala index fee6b8ac..e43b5f59 100644 --- a/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/repository/CategorizedRepository.scala +++ b/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/repository/CategorizedRepository.scala @@ -1,7 +1,5 @@ package com.karasiq.shadowcloud.storage.repository - - import akka.stream.scaladsl.Source import com.karasiq.shadowcloud.storage.repository.wrappers.{PrefixedRepositoryWrapper, RepositoryKeyMapper} @@ -15,7 +13,7 @@ trait CategorizedRepository[CatKey, ItemKey] extends Repository[(CatKey, ItemKey def subRepository(seq: CatKey): Repository[ItemKey] = { new RepositoryKeyMapper[(CatKey, ItemKey), ItemKey](this, _._2, (seq, _)) { override def keys: Source[ItemKey, Result] = CategorizedRepository.this.subKeys(seq) - override def toString: String = s"SubRepository($seq in ${CategorizedRepository.this})" + override def toString: String = s"SubRepository($seq in ${CategorizedRepository.this})" } } } diff --git a/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/repository/PathTreeRepository.scala b/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/repository/PathTreeRepository.scala index fbff5ba0..c497b5a2 100644 --- a/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/repository/PathTreeRepository.scala +++ b/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/repository/PathTreeRepository.scala @@ -31,21 +31,23 @@ object PathTreeRepository { def traverse(repository: PathTreeRepository, path: Path): PathTreeRepository = { new RepositoryKeyMapper[Path, Path](repository, _.toRelative(path), path / _) with PathTreeRepository { override def subKeys(fromPath: Path): Source[Path, Result] = repository.subKeys(path / fromPath) - override def keys: Source[Path, Result] = repository.subKeys(path) - override def toString: String = s"Traverse($repository [$path])" + override def keys: Source[Path, Result] = repository.subKeys(path) + override def toString: String = s"Traverse($repository [$path])" } } def toCategorized(repository: PathTreeRepository, path: Path = Path.root): CategorizedRepository[String, String] = { - new RepositoryKeyMapper[Path, (String, String)](repository, - path ⇒ (path.parent.name, path.name), { case (s1, s2) ⇒ path / s1 / s2 }) with CategorizedRepository[String, String] { + new RepositoryKeyMapper[Path, (String, String)](repository, path ⇒ (path.parent.name, path.name), { case (s1, s2) ⇒ path / s1 / s2 }) + with CategorizedRepository[String, String] { - override def keys: Source[(String, String), Result] = repository.subKeys(path) + override def keys: Source[(String, String), Result] = repository + .subKeys(path) .filter(_.nodes.length == 2) // .filterNot(p ⇒ p.isRoot || p.parent.isRoot) .map(path ⇒ (path.parent.name, path.name)) override def subKeys(seq: String): Source[String, Result] = { - repository.subKeys(path / seq) + repository + .subKeys(path / seq) .filter(_.nodes.length == 1) .map(_.name) } @@ -58,7 +60,8 @@ object PathTreeRepository { def toKeyValue(repository: PathTreeRepository, path: Path = Path.root): KeyValueRepository = { new RepositoryKeyMapper[Path, String](repository, _.name, path / _) { - override def keys: Source[String, Result] = repository.subKeys(path) + override def keys: Source[String, Result] = repository + .subKeys(path) .filter(_.nodes.length == 1) .map(_.name) @@ -78,4 +81,4 @@ object PathTreeRepository { .map(_.toRelative(fromPath)) .named("extractSubPaths") } -} \ No newline at end of file +} diff --git a/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/repository/Repository.scala b/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/repository/Repository.scala index 55b7182b..19093706 100644 --- a/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/repository/Repository.scala +++ b/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/repository/Repository.scala @@ -10,10 +10,15 @@ import akka.util.ByteString import com.karasiq.common.encoding.{ByteStringEncoding, HexString} import com.karasiq.shadowcloud.model.{ChunkId, SequenceNr} import com.karasiq.shadowcloud.storage.StorageIOResult -import com.karasiq.shadowcloud.storage.repository.wrappers.{CategorizedRepositoryKeyMapper, RepositoryKeyMapper, RepositoryWrapper, SubRepositoriesWrapper} +import com.karasiq.shadowcloud.storage.repository.wrappers.{ + CategorizedRepositoryKeyMapper, + RepositoryKeyMapper, + RepositoryWrapper, + SubRepositoriesWrapper +} trait Repository[Key] { - final type Data = ByteString + final type Data = ByteString final type Result = Future[StorageIOResult] def keys: Source[Key, Result] @@ -23,23 +28,30 @@ trait Repository[Key] { } object Repository { - def mapKeys[OldKey, NewKey](repository: Repository[OldKey], toNew: OldKey ⇒ NewKey, - toOld: NewKey ⇒ OldKey): Repository[NewKey] = { + def mapKeys[OldKey, NewKey](repository: Repository[OldKey], toNew: OldKey ⇒ NewKey, toOld: NewKey ⇒ OldKey): Repository[NewKey] = { new RepositoryKeyMapper(repository, toNew, toOld) } - def mapCategoryKeys[OldKey, NewKey, ItemKey](repository: CategorizedRepository[OldKey, ItemKey], toNew: OldKey ⇒ NewKey, - toOld: NewKey ⇒ OldKey): CategorizedRepository[NewKey, ItemKey] = { + def mapCategoryKeys[OldKey, NewKey, ItemKey]( + repository: CategorizedRepository[OldKey, ItemKey], + toNew: OldKey ⇒ NewKey, + toOld: NewKey ⇒ OldKey + ): CategorizedRepository[NewKey, ItemKey] = { new CategorizedRepositoryKeyMapper[OldKey, ItemKey, NewKey, ItemKey](repository, toNew, identity, toOld, identity) } - def mapItemKeys[CatKey, OldKey, NewKey](repository: CategorizedRepository[CatKey, OldKey], toNew: OldKey ⇒ NewKey, - toOld: NewKey ⇒ OldKey): CategorizedRepository[CatKey, NewKey] = { + def mapItemKeys[CatKey, OldKey, NewKey]( + repository: CategorizedRepository[CatKey, OldKey], + toNew: OldKey ⇒ NewKey, + toOld: NewKey ⇒ OldKey + ): CategorizedRepository[CatKey, NewKey] = { new CategorizedRepositoryKeyMapper[CatKey, OldKey, CatKey, NewKey](repository, identity, toNew, identity, toOld) } - def forChunks[CatKey](repository: CategorizedRepository[CatKey, String], - encoding: ByteStringEncoding = HexString): CategorizedRepository[CatKey, ChunkId] = { + def forChunks[CatKey]( + repository: CategorizedRepository[CatKey, String], + encoding: ByteStringEncoding = HexString + ): CategorizedRepository[CatKey, ChunkId] = { mapItemKeys(repository, encoding.decode, encoding.encode) } @@ -63,8 +75,10 @@ object Repository { new RepositoryWrapper(repository) with CategorizedRepository[CatKey, Key] } - def fromSubRepositories[CatKey, Key](pathString: String, subRepositories: () ⇒ Source[(CatKey, Repository[Key]), NotUsed]) - (implicit ec: ExecutionContext, mat: Materializer): CategorizedRepository[CatKey, Key] = { + def fromSubRepositories[CatKey, Key](pathString: String, subRepositories: () ⇒ Source[(CatKey, Repository[Key]), NotUsed])(implicit + ec: ExecutionContext, + mat: Materializer + ): CategorizedRepository[CatKey, Key] = { new SubRepositoriesWrapper(pathString, subRepositories) } diff --git a/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/repository/RepositoryKeys.scala b/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/repository/RepositoryKeys.scala index db7d9636..4b622f89 100644 --- a/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/repository/RepositoryKeys.scala +++ b/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/repository/RepositoryKeys.scala @@ -1,7 +1,5 @@ package com.karasiq.shadowcloud.storage.repository - - import akka.util.ByteString import com.karasiq.common.encoding.HexString import com.karasiq.shadowcloud.storage.repository.wrappers.LongSeqRepositoryWrapper diff --git a/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/repository/RepositoryStreams.scala b/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/repository/RepositoryStreams.scala index 09edca41..7e61cb32 100644 --- a/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/repository/RepositoryStreams.scala +++ b/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/repository/RepositoryStreams.scala @@ -38,7 +38,8 @@ object RepositoryStreams { def readWithRetries[K](repository: Repository[K], key: K, retries: Int = DefaultRetries): Source[repository.Data, repository.Result] = { def openStream() = { val promise = Promise[StorageIOResult]() - repository.read(key) + repository + .read(key) .mapMaterializedValue(promise.completeWith) .alsoTo(Sink.onComplete(_.failed.foreach(promise.tryFailure))) .via(AkkaStreamUtils.extractUpstream) @@ -46,7 +47,8 @@ object RepositoryStreams { .mapMaterializedValue(_ ⇒ NotUsed) } - openStream().recoverWithRetries(retries, { case _ ⇒ openStream() }) + openStream() + .recoverWithRetries(retries, { case _ ⇒ openStream() }) .alsoToMat { Flow[(Source[ByteString, NotUsed], Future[StorageIOResult])] .flatMapConcat { case (_, future) ⇒ Source.future(future) } diff --git a/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/repository/wrappers/CategorizedRepositoryKeyMapper.scala b/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/repository/wrappers/CategorizedRepositoryKeyMapper.scala index cd82c153..3390680b 100644 --- a/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/repository/wrappers/CategorizedRepositoryKeyMapper.scala +++ b/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/repository/wrappers/CategorizedRepositoryKeyMapper.scala @@ -2,13 +2,19 @@ package com.karasiq.shadowcloud.storage.repository.wrappers import com.karasiq.shadowcloud.storage.repository.CategorizedRepository -private[repository] class CategorizedRepositoryKeyMapper[OldCK, OldIK, NewCK, NewIK] - (repository: CategorizedRepository[OldCK, OldIK], toNewCK: OldCK ⇒ NewCK, toNewIK: OldIK ⇒ NewIK, toOldCK: NewCK ⇒ OldCK, toOldIK: NewIK ⇒ OldIK) - extends RepositoryKeyMapper[(OldCK, OldIK), (NewCK, NewIK)](repository, - key ⇒ key.copy(toNewCK(key._1), toNewIK(key._2)), - key ⇒ key.copy(toOldCK(key._1), toOldIK(key._2))) with CategorizedRepository[NewCK, NewIK] { - +private[repository] class CategorizedRepositoryKeyMapper[OldCK, OldIK, NewCK, NewIK]( + repository: CategorizedRepository[OldCK, OldIK], + toNewCK: OldCK ⇒ NewCK, + toNewIK: OldIK ⇒ NewIK, + toOldCK: NewCK ⇒ OldCK, + toOldIK: NewIK ⇒ OldIK +) extends RepositoryKeyMapper[(OldCK, OldIK), (NewCK, NewIK)]( + repository, + key ⇒ key.copy(toNewCK(key._1), toNewIK(key._2)), + key ⇒ key.copy(toOldCK(key._1), toOldIK(key._2)) + ) + with CategorizedRepository[NewCK, NewIK] { override def subKeys(seq: NewCK) = repository.subKeys(toOldCK(seq)).map(toNewIK) - override def toString: String = s"CategorizedKeyMapper($repository)" + override def toString: String = s"CategorizedKeyMapper($repository)" } diff --git a/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/repository/wrappers/LongSeqRepositoryWrapper.scala b/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/repository/wrappers/LongSeqRepositoryWrapper.scala index 1d5cb45e..2c5ef135 100644 --- a/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/repository/wrappers/LongSeqRepositoryWrapper.scala +++ b/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/repository/wrappers/LongSeqRepositoryWrapper.scala @@ -5,8 +5,8 @@ import akka.stream.scaladsl.{Flow, Keep, Sink, Source} import com.karasiq.shadowcloud.storage.repository.{KeyValueRepository, SeqRepository} private[repository] final class LongSeqRepositoryWrapper(underlying: KeyValueRepository) extends SeqRepository[Long] { - def keys: Source[Long, Result] = underlying.keys.map(_.toLong) + def keys: Source[Long, Result] = underlying.keys.map(_.toLong) def read(key: Long): Source[Data, Result] = underlying.read(key.toString) - def write(key: Long): Sink[Data, Result] = underlying.write(key.toString) - def delete: Sink[Long, Result] = Flow[Long].map(_.toString).toMat(underlying.delete)(Keep.right) + def write(key: Long): Sink[Data, Result] = underlying.write(key.toString) + def delete: Sink[Long, Result] = Flow[Long].map(_.toString).toMat(underlying.delete)(Keep.right) } diff --git a/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/repository/wrappers/PathNodesMapper.scala b/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/repository/wrappers/PathNodesMapper.scala index 77da224e..570025a1 100644 --- a/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/repository/wrappers/PathNodesMapper.scala +++ b/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/repository/wrappers/PathNodesMapper.scala @@ -8,14 +8,14 @@ import com.karasiq.shadowcloud.model.Path import com.karasiq.shadowcloud.storage.repository.PathTreeRepository class PathNodesMapper(repository: PathTreeRepository, toNew: String ⇒ String, toOld: String ⇒ String) extends PathTreeRepository { - override def keys: Source[Path, Result] = repository.keys.map(toCleanPath) + override def keys: Source[Path, Result] = repository.keys.map(toCleanPath) def subKeys(fromPath: Path): Source[Path, Result] = repository.subKeys(toStoragePath(fromPath)).map(toCleanPath) - def read(key: Path): Source[Data, Result] = repository.read(toStoragePath(key)) - def write(key: Path): Sink[Data, Result] = repository.write(toStoragePath(key)) - def delete: Sink[Path, Result] = Flow[Path].map(toStoragePath).toMat(repository.delete)(Keep.right) + def read(key: Path): Source[Data, Result] = repository.read(toStoragePath(key)) + def write(key: Path): Sink[Data, Result] = repository.write(toStoragePath(key)) + def delete: Sink[Path, Result] = Flow[Path].map(toStoragePath).toMat(repository.delete)(Keep.right) private[this] def toStoragePath(path: Path) = Path(path.nodes.map(toNew)) - private[this] def toCleanPath(path: Path) = Path(path.nodes.map(toOld)) + private[this] def toCleanPath(path: Path) = Path(path.nodes.map(toOld)) } object PathNodesMapper { @@ -29,8 +29,6 @@ object PathNodesMapper { } def encode(repository: PathTreeRepository, encoding: ByteStringEncoding): PathTreeRepository = { - apply(repository, - str ⇒ encoding.encode(ByteString(str)), - str ⇒ encoding.decode(str).utf8String) + apply(repository, str ⇒ encoding.encode(ByteString(str)), str ⇒ encoding.decode(str).utf8String) } } diff --git a/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/repository/wrappers/PathStringRepositoryWrapper.scala b/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/repository/wrappers/PathStringRepositoryWrapper.scala index d311e252..7d4989f5 100644 --- a/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/repository/wrappers/PathStringRepositoryWrapper.scala +++ b/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/repository/wrappers/PathStringRepositoryWrapper.scala @@ -6,8 +6,12 @@ import com.karasiq.shadowcloud.model.Path import com.karasiq.shadowcloud.storage.repository.{KeyValueRepository, PathTreeRepository} private[repository] final class PathStringRepositoryWrapper(repository: KeyValueRepository, delimiter: String) - extends RepositoryKeyMapper[String, Path](repository, PathStringRepositoryWrapper.split(_, delimiter), - PathStringRepositoryWrapper.combine(_, delimiter)) with PathTreeRepository { + extends RepositoryKeyMapper[String, Path]( + repository, + PathStringRepositoryWrapper.split(_, delimiter), + PathStringRepositoryWrapper.combine(_, delimiter) + ) + with PathTreeRepository { def subKeys(fromPath: Path): Source[Path, Result] = { repository.keys @@ -30,4 +34,4 @@ private object PathStringRepositoryWrapper { require(key.nodes.forall(n ⇒ !n.contains(delimiter)), "Cannot map path to string: " + key) key.nodes.mkString(delimiter) } -} \ No newline at end of file +} diff --git a/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/repository/wrappers/PrefixedRepositoryWrapper.scala b/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/repository/wrappers/PrefixedRepositoryWrapper.scala index 6f011e7e..75bedbdb 100644 --- a/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/repository/wrappers/PrefixedRepositoryWrapper.scala +++ b/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/repository/wrappers/PrefixedRepositoryWrapper.scala @@ -3,8 +3,12 @@ package com.karasiq.shadowcloud.storage.repository.wrappers import com.karasiq.shadowcloud.storage.repository.{CategorizedRepository, KeyValueRepository} private[repository] final class PrefixedRepositoryWrapper(repository: KeyValueRepository, delimiter: String) - extends RepositoryKeyMapper[String, (String, String)](repository, PrefixedRepositoryWrapper.split(_, delimiter), - PrefixedRepositoryWrapper.combine(_, delimiter)) with CategorizedRepository[String, String] + extends RepositoryKeyMapper[String, (String, String)]( + repository, + PrefixedRepositoryWrapper.split(_, delimiter), + PrefixedRepositoryWrapper.combine(_, delimiter) + ) + with CategorizedRepository[String, String] private object PrefixedRepositoryWrapper { def split(str: String, delimiter: String): (String, String) = { @@ -16,4 +20,4 @@ private object PrefixedRepositoryWrapper { val (seqKey, itemKey) = key s"$seqKey$delimiter$itemKey" } -} \ No newline at end of file +} diff --git a/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/repository/wrappers/RepositoryKeyMapper.scala b/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/repository/wrappers/RepositoryKeyMapper.scala index e4bce86c..38911b2f 100644 --- a/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/repository/wrappers/RepositoryKeyMapper.scala +++ b/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/repository/wrappers/RepositoryKeyMapper.scala @@ -1,20 +1,19 @@ package com.karasiq.shadowcloud.storage.repository.wrappers - import akka.stream.scaladsl.{Flow, Keep, Sink, Source} import com.karasiq.shadowcloud.storage.repository.Repository import scala.util.Try -private[repository] class RepositoryKeyMapper[OldKey, NewKey](repository: Repository[OldKey], toNew: OldKey ⇒ NewKey, - toOld: NewKey ⇒ OldKey) extends Repository[NewKey] { +private[repository] class RepositoryKeyMapper[OldKey, NewKey](repository: Repository[OldKey], toNew: OldKey ⇒ NewKey, toOld: NewKey ⇒ OldKey) + extends Repository[NewKey] { def keys: Source[NewKey, Result] = repository.keys .mapConcat(key ⇒ Try(toNew(key)).toOption.toList) .named("mappedKeys") - + def read(key: NewKey): Source[Data, Result] = repository.read(toOld(key)) - def write(key: NewKey): Sink[Data, Result] = repository.write(toOld(key)) - def delete: Sink[NewKey, Result] = Flow[NewKey].map(toOld).toMat(repository.delete)(Keep.right) + def write(key: NewKey): Sink[Data, Result] = repository.write(toOld(key)) + def delete: Sink[NewKey, Result] = Flow[NewKey].map(toOld).toMat(repository.delete)(Keep.right) override def toString: String = { s"KeyMapper($repository)" diff --git a/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/repository/wrappers/RepositoryWrapper.scala b/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/repository/wrappers/RepositoryWrapper.scala index 8586a8d5..a03e8e70 100644 --- a/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/repository/wrappers/RepositoryWrapper.scala +++ b/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/repository/wrappers/RepositoryWrapper.scala @@ -1,11 +1,8 @@ package com.karasiq.shadowcloud.storage.repository.wrappers - - import com.karasiq.shadowcloud.storage.repository.Repository -private[repository] class RepositoryWrapper[Key](underlying: Repository[Key]) - extends RepositoryKeyMapper[Key, Key](underlying, identity, identity) { +private[repository] class RepositoryWrapper[Key](underlying: Repository[Key]) extends RepositoryKeyMapper[Key, Key](underlying, identity, identity) { override def toString: String = { underlying.toString diff --git a/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/repository/wrappers/SubRepositoriesWrapper.scala b/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/repository/wrappers/SubRepositoriesWrapper.scala index 8dcaac5e..312f7f84 100644 --- a/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/repository/wrappers/SubRepositoriesWrapper.scala +++ b/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/repository/wrappers/SubRepositoriesWrapper.scala @@ -10,11 +10,12 @@ import com.karasiq.shadowcloud.streams.utils.AkkaStreamUtils import scala.concurrent.{ExecutionContext, Future, Promise} import scala.util.{Failure, Success} -private[repository] final class SubRepositoriesWrapper[CatKey, Key](pathString: String, - subRepositories: () ⇒ Source[(CatKey, Repository[Key]), NotUsed]) - (implicit ec: ExecutionContext) - extends CategorizedRepository[CatKey, Key] { - +private[repository] final class SubRepositoriesWrapper[CatKey, Key]( + pathString: String, + subRepositories: () ⇒ Source[(CatKey, Repository[Key]), NotUsed] +)(implicit ec: ExecutionContext) + extends CategorizedRepository[CatKey, Key] { + override def subRepository(categoryKey: CatKey): Repository[Key] = { new Repository[Key] { private[this] def openSubStream[T, M](extractSource: Repository[Key] ⇒ Source[T, M]): Source[T, Future[M]] = { @@ -34,7 +35,7 @@ private[repository] final class SubRepositoriesWrapper[CatKey, Key](pathString: } def keys: Source[Key, Result] = { - val promise = Promise[StorageIOResult] + val promise = Promise[StorageIOResult] val repositories = subRepositories() repositories .filter(_._1 == categoryKey) @@ -80,7 +81,7 @@ private[repository] final class SubRepositoriesWrapper[CatKey, Key](pathString: Flow[(Source[Key, NotUsed], Result)] .mapAsync(1)(_._2) .fold(Seq.empty[StorageIOResult])(_ :+ _) - .map(results ⇒ StorageUtils.foldIOResultsIgnoreErrors(results:_*)) + .map(results ⇒ StorageUtils.foldIOResultsIgnoreErrors(results: _*)) .toMat(Sink.head)(Keep.right) )(Keep.right) .flatMapConcat(_._1) @@ -90,7 +91,7 @@ private[repository] final class SubRepositoriesWrapper[CatKey, Key](pathString: } def keys: Source[(CatKey, Key), Result] = { - val promise = Promise[StorageIOResult] + val promise = Promise[StorageIOResult] val repositories = subRepositories() repositories .flatMapConcat { case (ck, repo) ⇒ @@ -119,9 +120,10 @@ private[repository] final class SubRepositoriesWrapper[CatKey, Key](pathString: .groupBy(100, _._1) .prefixAndTail(1) .map { case (head +: Nil, source) ⇒ - val promise = Promise[StorageIOResult] + val promise = Promise[StorageIOResult] val repository = subRepository(head._1) - val stream = Source.single(head) + val stream = Source + .single(head) .concat(source) .map(_._2) .alsoToMat(repository.delete)(Keep.right) @@ -134,7 +136,7 @@ private[repository] final class SubRepositoriesWrapper[CatKey, Key](pathString: Flow[(Source[Key, NotUsed], Result)] .mapAsync(1)(_._2) .fold(Seq.empty[StorageIOResult])(_ :+ _) - .map(results ⇒ StorageUtils.foldIOResultsIgnoreErrors(results:_*)) + .map(results ⇒ StorageUtils.foldIOResultsIgnoreErrors(results: _*)) .toMat(Sink.head)(Keep.right) )(Keep.right) .flatMapConcat(_._1) diff --git a/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/utils/StorageUtils.scala b/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/utils/StorageUtils.scala index ba1cf8eb..0b19f1de 100644 --- a/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/utils/StorageUtils.scala +++ b/storage/parent/src/main/scala/com/karasiq/shadowcloud/storage/utils/StorageUtils.scala @@ -4,7 +4,7 @@ import java.io.FileNotFoundException import java.nio.file.FileAlreadyExistsException import akka.stream.scaladsl.Flow -import akka.stream.{IOResult => AkkaIOResult} +import akka.stream.{IOResult ⇒ AkkaIOResult} import akka.util.ByteString import akka.{Done, NotUsed} import com.karasiq.common.encoding.HexString @@ -77,7 +77,7 @@ object StorageUtils { def wrapStream(path: Path = Path.root): Flow[StorageIOResult, StorageIOResult, NotUsed] = { Flow[StorageIOResult] .recover { case error ⇒ failure(path, error) } - // .orElse(Source.single(StorageIOResult.Failure(path, StorageUtils.wrapException(path, new IllegalArgumentException("No data passed"))))) + // .orElse(Source.single(StorageIOResult.Failure(path, StorageUtils.wrapException(path, new IllegalArgumentException("No data passed"))))) } def foldStream(path: Path = Path.root): Flow[StorageIOResult, StorageIOResult, NotUsed] = { @@ -121,16 +121,14 @@ object StorageUtils { def foldIOResultsIgnoreErrors(results: StorageIOResult*): StorageIOResult = { if (results.isEmpty) return StorageIOResult.empty - val path = results.headOption.fold(Path.root)(_.path) - val count = results - .collect { case StorageIOResult.Success(_, count) ⇒ count } - .sum + val path = results.headOption.fold(Path.root)(_.path) + val count = results.collect { case StorageIOResult.Success(_, count) ⇒ count }.sum StorageIOResult.Success(path, count) } def foldIOResults(results: StorageIOResult*): StorageIOResult = { if (results.isEmpty) return StorageIOResult.empty - results.find(_.isFailure).getOrElse(foldIOResultsIgnoreErrors(results:_*)) + results.find(_.isFailure).getOrElse(foldIOResultsIgnoreErrors(results: _*)) } def foldIOFutures(fs: Future[StorageIOResult]*)(implicit ec: ExecutionContext): Future[StorageIOResult] = { diff --git a/storage/telegram/src/main/scala/com/karasiq/shadowcloud/storage/telegram/TelegramRepository.scala b/storage/telegram/src/main/scala/com/karasiq/shadowcloud/storage/telegram/TelegramRepository.scala index eb8fbce9..92b93751 100644 --- a/storage/telegram/src/main/scala/com/karasiq/shadowcloud/storage/telegram/TelegramRepository.scala +++ b/storage/telegram/src/main/scala/com/karasiq/shadowcloud/storage/telegram/TelegramRepository.scala @@ -87,13 +87,12 @@ class TelegramRepository(port: Int)(implicit as: ActorSystem) extends PathTreeRe )(Keep.right) // .via(ByteStreams.concat) .via(AkkaStreamUtils.extractUpstream) - .map( - upstream ⇒ - HttpRequest( - HttpMethods.POST, - Uri("/upload").withQuery(Uri.Query("path" → encodePath(path))), - entity = HttpEntity(ContentTypes.`application/octet-stream`, upstream) - ) + .map(upstream ⇒ + HttpRequest( + HttpMethods.POST, + Uri("/upload").withQuery(Uri.Query("path" → encodePath(path))), + entity = HttpEntity(ContentTypes.`application/octet-stream`, upstream) + ) ) .via(executeHttpRequest) .alsoTo(Sink.foreach(_.discardEntityBytes())) @@ -103,11 +102,11 @@ class TelegramRepository(port: Int)(implicit as: ActorSystem) extends PathTreeRe case r ⇒ StorageIOResult.Failure(path, StorageUtils.wrapException(path, new IOException(s"Request error: $r"))) } .via(StorageUtils.wrapStream(path)) - .toMat(Sink.headOption) { case (bytesCount, reqResult) => + .toMat(Sink.headOption) { case (bytesCount, reqResult) ⇒ val result = reqResult.map(_.getOrElse(StorageUtils.failure(path, new IOException))) StorageUtils.foldIOFutures(result, bytesCount) } - //.toMat(Sink.headOption)(Keep.right(_, _).map(_.getOrElse(StorageUtils.failure(path, new IOException)))) + //.toMat(Sink.headOption)(Keep.right(_, _).map(_.getOrElse(StorageUtils.failure(path, new IOException)))) } override def delete: Sink[Path, Result] = diff --git a/storage/webdav/src/main/scala/com/karasiq/shadowcloud/webdav/SardineRepository.scala b/storage/webdav/src/main/scala/com/karasiq/shadowcloud/webdav/SardineRepository.scala index d99cc964..567e0d9b 100644 --- a/storage/webdav/src/main/scala/com/karasiq/shadowcloud/webdav/SardineRepository.scala +++ b/storage/webdav/src/main/scala/com/karasiq/shadowcloud/webdav/SardineRepository.scala @@ -32,9 +32,9 @@ object SardineRepository { new SardineRepository(props, sardine) } - def getResourceURL(baseUrl: String, path: Path): String = { + def getResourceURL(baseUrl: String, path: Path): String = { val urlWithSlash = if (baseUrl.endsWith("/")) baseUrl else baseUrl + "/" - val encodedPath = path.nodes.map(URLEncoder.encode(_, "UTF-8")) + val encodedPath = path.nodes.map(URLEncoder.encode(_, "UTF-8")) urlWithSlash + encodedPath.mkString("/") } @@ -45,7 +45,7 @@ object SardineRepository { val uri = new URI(if (ps.contains("://")) ps else "http://" + ps) val proxyType = uri.getScheme match { case "socks" | "socks4" | "socks5" ⇒ net.Proxy.Type.SOCKS - case _ ⇒ net.Proxy.Type.HTTP + case _ ⇒ net.Proxy.Type.HTTP } new net.Proxy(proxyType, InetSocketAddress.createUnresolved(uri.getHost, uri.getPort)) } @@ -53,14 +53,14 @@ object SardineRepository { } new ProxySelector { - def select(uri: URI): util.List[Proxy] = proxies.asJava + def select(uri: URI): util.List[Proxy] = proxies.asJava def connectFailed(uri: URI, sa: SocketAddress, ioe: IOException): Unit = () } } - val sardineConfig = props.rootConfig.getConfigIfExists("sardine") + val sardineConfig = props.rootConfig.getConfigIfExists("sardine") val maxConnections = sardineConfig.withDefault(8, _.getInt("max-connections")) - val proxySelector = createProxySelector(sardineConfig) + val proxySelector = createProxySelector(sardineConfig) val sardine = new SardineImpl(props.credentials.login, props.credentials.password, proxySelector) { override def createDefaultConnectionManager(schemeRegistry: Registry[ConnectionSocketFactory]): HttpClientConnectionManager = { @@ -77,12 +77,13 @@ object SardineRepository { } class SardineRepository(props: StorageProps, sardine: Sardine)(implicit dispatcher: MessageDispatcher) extends PathTreeRepository { - private[this] val rootUrl = props.address.uri.map(_.toString).getOrElse(throw new IllegalArgumentException("No WebDav URL")) - private[this] val baseUrl = SardineRepository.getResourceURL(rootUrl, props.address.path) + private[this] val rootUrl = props.address.uri.map(_.toString).getOrElse(throw new IllegalArgumentException("No WebDav URL")) + private[this] val baseUrl = SardineRepository.getResourceURL(rootUrl, props.address.path) private[this] val cachedDirectories = TrieMap.empty[Path, DavResource] def read(path: Path) = { - Source.single(path) + Source + .single(path) .viaMat(AkkaStreamUtils.flatMapConcatMat { path ⇒ val resourceUrl = SardineRepository.getResourceURL(baseUrl, path) StreamConverters.fromInputStream(() ⇒ sardine.get(resourceUrl)) @@ -121,7 +122,7 @@ class SardineRepository(props: StorageProps, sardine: Sardine)(implicit dispatch Flow[Path] .map { path ⇒ val resourceUrl = SardineRepository.getResourceURL(baseUrl, path) - val size = sardine.list(resourceUrl, 0).asScala.head.getContentLength + val size = sardine.list(resourceUrl, 0).asScala.head.getContentLength sardine.delete(resourceUrl) StorageIOResult.Success(path, size) } @@ -161,7 +162,8 @@ class SardineRepository(props: StorageProps, sardine: Sardine)(implicit dispatch .named("webdavTraverse") } - Source.single(fromPath) + Source + .single(fromPath) .via(traverseDirectory) .map(_.toRelative(fromPath)) .alsoToMat(StorageUtils.countPassedElements(fromPath).toMat(Sink.head)(Keep.right))(Keep.right) diff --git a/storage/webdav/src/main/scala/com/karasiq/shadowcloud/webdav/WebDavStoragePlugin.scala b/storage/webdav/src/main/scala/com/karasiq/shadowcloud/webdav/WebDavStoragePlugin.scala index d0fd38a1..b9ad781d 100644 --- a/storage/webdav/src/main/scala/com/karasiq/shadowcloud/webdav/WebDavStoragePlugin.scala +++ b/storage/webdav/src/main/scala/com/karasiq/shadowcloud/webdav/WebDavStoragePlugin.scala @@ -17,8 +17,8 @@ object WebDavStoragePlugin { class WebDavStoragePlugin extends StoragePlugin { def createStorage(storageId: StorageId, props: StorageProps)(implicit context: ActorContext) = { - implicit val ec = context.system.dispatchers.lookup(WebDavStoragePlugin.dispatcherId) - val sardine = SardineRepository.createSardine(props) + implicit val ec = context.system.dispatchers.lookup(WebDavStoragePlugin.dispatcherId) + val sardine = SardineRepository.createSardine(props) val repository: SardineRepository = SardineRepository(props, sardine) StoragePluginBuilder(storageId, props) .withChunksTree(repository) diff --git a/storage/webdav/src/main/scala/com/karasiq/shadowcloud/webdav/WebDavStorageProvider.scala b/storage/webdav/src/main/scala/com/karasiq/shadowcloud/webdav/WebDavStorageProvider.scala index d0b7e82b..45961ac5 100644 --- a/storage/webdav/src/main/scala/com/karasiq/shadowcloud/webdav/WebDavStorageProvider.scala +++ b/storage/webdav/src/main/scala/com/karasiq/shadowcloud/webdav/WebDavStorageProvider.scala @@ -11,8 +11,7 @@ class WebDavStorageProvider extends StorageProvider { WebDavStoragePlugin() } - override def storageConfigs = { - case "webdav" ⇒ - ConfigProps("type" → "webdav", "address.uri" → "https://example.com/dir", "credentials.login" → "example", "credentials.password" → "123456") + override def storageConfigs = { case "webdav" ⇒ + ConfigProps("type" → "webdav", "address.uri" → "https://example.com/dir", "credentials.login" → "example", "credentials.password" → "123456") } } diff --git a/storage/yandex/src/main/scala/com/karasiq/shadowcloud/storage/yandex/YandexStoragePlugin.scala b/storage/yandex/src/main/scala/com/karasiq/shadowcloud/storage/yandex/YandexStoragePlugin.scala index bdb547aa..a990613a 100644 --- a/storage/yandex/src/main/scala/com/karasiq/shadowcloud/storage/yandex/YandexStoragePlugin.scala +++ b/storage/yandex/src/main/scala/com/karasiq/shadowcloud/storage/yandex/YandexStoragePlugin.scala @@ -18,15 +18,18 @@ class YandexStoragePlugin extends StoragePlugin { override def createStorage(storageId: StorageId, props: StorageProps)(implicit context: ActorContext): ActorRef = { import context.{dispatcher, system} val sc = ShadowCloud() - val api = new YandexWebApi(solveCaptcha = { imageUrl ⇒ - sc.challenges - .create(s"Yandex captcha ($storageId)", s"""""") - .map(_.utf8String) - }, passChallenge = { url ⇒ - sc.challenges - .create(s"Yandex verification ($storageId)", s"""Please complete the verification""") - .map(_ ⇒ Done) - }) + val api = new YandexWebApi( + solveCaptcha = { imageUrl ⇒ + sc.challenges + .create(s"Yandex captcha ($storageId)", s"""""") + .map(_.utf8String) + }, + passChallenge = { url ⇒ + sc.challenges + .create(s"Yandex verification ($storageId)", s"""Please complete the verification""") + .map(_ ⇒ Done) + } + ) implicit val session: YandexSession = { def create() = Await.result(api.createSession(props.credentials.login, props.credentials.password), 5 minutes) @@ -50,9 +53,8 @@ class YandexStoragePlugin extends StoragePlugin { .withIndexTree(repository) .withHealth(new StorageHealthProvider { override def health: Future[StorageHealth] = - api.usedSpace().map { - case YandexUsedSpace(used, free, limit) ⇒ - StorageHealth.normalized(free, limit, used) + api.usedSpace().map { case YandexUsedSpace(used, free, limit) ⇒ + StorageHealth.normalized(free, limit, used) } }) .createStorage() diff --git a/storage/yandex/src/main/scala/com/karasiq/shadowcloud/storage/yandex/YandexStorageProvider.scala b/storage/yandex/src/main/scala/com/karasiq/shadowcloud/storage/yandex/YandexStorageProvider.scala index bb0888f2..b48ede13 100644 --- a/storage/yandex/src/main/scala/com/karasiq/shadowcloud/storage/yandex/YandexStorageProvider.scala +++ b/storage/yandex/src/main/scala/com/karasiq/shadowcloud/storage/yandex/YandexStorageProvider.scala @@ -7,15 +7,15 @@ class YandexStorageProvider extends StorageProvider { override def storageTypes: Set[String] = Set("yandex") override def storages: StoragePF = { - case props if props.storageType == "yandex" => + case props if props.storageType == "yandex" ⇒ new YandexStoragePlugin } - override def storageConfigs: StorageConfigPF = { - case "yandex" => ConfigProps( - "type" -> "yandex", - "credentials.login" -> "johndoe123", - "credentials.password" -> "passw0rd" + override def storageConfigs: StorageConfigPF = { case "yandex" ⇒ + ConfigProps( + "type" → "yandex", + "credentials.login" → "johndoe123", + "credentials.password" → "passw0rd" ) } } diff --git a/storage/yandex/src/main/scala/com/karasiq/shadowcloud/storage/yandex/YandexWebApi.scala b/storage/yandex/src/main/scala/com/karasiq/shadowcloud/storage/yandex/YandexWebApi.scala index 9fc11bd8..8a007534 100644 --- a/storage/yandex/src/main/scala/com/karasiq/shadowcloud/storage/yandex/YandexWebApi.scala +++ b/storage/yandex/src/main/scala/com/karasiq/shadowcloud/storage/yandex/YandexWebApi.scala @@ -179,19 +179,17 @@ private[yandex] class YandexWebApi( http .singleRequest(HttpRequest(HttpMethods.GET, passport, stdHeaders), settings = poolSettings) .flatMap(entityWithCookies) - .map { - case (entity, cookies) ⇒ - val str = entity.data.utf8String - val csrf = csrfR.findFirstMatchIn(str).map(_.group("csrf")).get - val process = proccessR.findFirstMatchIn(str).map(_.group("uuid")).get - (csrf, process, cookies) + .map { case (entity, cookies) ⇒ + val str = entity.data.utf8String + val csrf = csrfR.findFirstMatchIn(str).map(_.group("csrf")).get + val process = proccessR.findFirstMatchIn(str).map(_.group("uuid")).get + (csrf, process, cookies) } } private[this] def entityWithCookies(resp: HttpResponse) = { - val cookies = resp.headers.collect { - case `Set-Cookie`(cookie) ⇒ - cookie + val cookies = resp.headers.collect { case `Set-Cookie`(cookie) ⇒ + cookie } resp.entity.toStrict(5 seconds).map(_ → cookies) } @@ -292,38 +290,35 @@ private[yandex] class YandexWebApi( .flatMap(entityWithCookies) executePasswordRequest() - .flatMap { - case (response, cookies) ⇒ - val json = Json.parse(response.data.toArray) - if ((json \ "errors" \ 0).asOpt[String].contains("captcha.required")) { - for { - (image, key) ← getCaptcha() - answer ← solveCaptcha(image) - Done ← checkCaptcha(key, answer) - result ← executePasswordRequest() - } yield result - } else if ((json \ "redirect_url").asOpt[String].isDefined) { - val url = s"https://passport.yandex.ru${(json \ "redirect_url").as[String]}" - passChallenge(url).map(_ ⇒ (response → cookies)) - } else { - Future.successful(response → cookies) - } + .flatMap { case (response, cookies) ⇒ + val json = Json.parse(response.data.toArray) + if ((json \ "errors" \ 0).asOpt[String].contains("captcha.required")) { + for { + (image, key) ← getCaptcha() + answer ← solveCaptcha(image) + Done ← checkCaptcha(key, answer) + result ← executePasswordRequest() + } yield result + } else if ((json \ "redirect_url").asOpt[String].isDefined) { + val url = s"https://passport.yandex.ru${(json \ "redirect_url").as[String]}" + passChallenge(url).map(_ ⇒ (response → cookies)) + } else { + Future.successful(response → cookies) + } } - .flatMap { - case (response, sessCookies) ⇒ - val json = Json.parse(response.data.toArray) - require((json \ "status").as[String] == "ok", s"Not ok: $json") - val validateUrl = "https://passport.yandex.ru/registration-validations/auth/accounts" - val headers = List( - Cookie((cookies ++ sessCookies).map(_.pair()).toList) - ) - http - .singleRequest(HttpRequest(HttpMethods.POST, validateUrl, headers, FormData("csrf_token" → csrf).toEntity), settings = poolSettings) - .flatMap(entityWithCookies) - .map { - case (entity, authCookies) ⇒ - Json.parse(entity.data.toArray).as[JsObject] → (cookies ++ sessCookies ++ authCookies) - } + .flatMap { case (response, sessCookies) ⇒ + val json = Json.parse(response.data.toArray) + require((json \ "status").as[String] == "ok", s"Not ok: $json") + val validateUrl = "https://passport.yandex.ru/registration-validations/auth/accounts" + val headers = List( + Cookie((cookies ++ sessCookies).map(_.pair()).toList) + ) + http + .singleRequest(HttpRequest(HttpMethods.POST, validateUrl, headers, FormData("csrf_token" → csrf).toEntity), settings = poolSettings) + .flatMap(entityWithCookies) + .map { case (entity, authCookies) ⇒ + Json.parse(entity.data.toArray).as[JsObject] → (cookies ++ sessCookies ++ authCookies) + } } } @@ -343,12 +338,11 @@ private[yandex] class YandexWebApi( http .singleRequest(HttpRequest(HttpMethods.GET, "https://disk.yandex.ru/client/disk", headers ++ userAgentHeaders), settings = poolSettings) .flatMap(entityWithCookies) - .map { - case (entity, _) ⇒ - regex - .findFirstMatchIn(entity.data.utf8String) - .map(_.group("sk")) - .getOrElse(throw new IllegalArgumentException("No sk found")) + .map { case (entity, _) ⇒ + regex + .findFirstMatchIn(entity.data.utf8String) + .map(_.group("sk")) + .getOrElse(throw new IllegalArgumentException("No sk found")) } } diff --git a/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/actors/SCDispatchers.scala b/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/actors/SCDispatchers.scala index c72677d2..83fd1592 100644 --- a/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/actors/SCDispatchers.scala +++ b/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/actors/SCDispatchers.scala @@ -1,7 +1,7 @@ package com.karasiq.shadowcloud.actors private[shadowcloud] object SCDispatchers { - val metadata = "shadowcloud.metadata.default-dispatcher" + val metadata = "shadowcloud.metadata.default-dispatcher" val metadataBlocking = "shadowcloud.metadata.blocking-dispatcher" - val cryptography = "shadowcloud.crypto.default-dispatcher" + val cryptography = "shadowcloud.crypto.default-dispatcher" } diff --git a/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/actors/utils/ContainerActor.scala b/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/actors/utils/ContainerActor.scala index 84a494f1..419f0e1c 100644 --- a/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/actors/utils/ContainerActor.scala +++ b/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/actors/utils/ContainerActor.scala @@ -82,7 +82,7 @@ private[actors] trait ContainerActor { self: Actor with Stash with ActorLogging } } - override def supervisorStrategy: _root_.akka.actor.SupervisorStrategy = OneForOneStrategy() { - case NonFatal(_) ⇒ SupervisorStrategy.Stop + override def supervisorStrategy: _root_.akka.actor.SupervisorStrategy = OneForOneStrategy() { case NonFatal(_) ⇒ + SupervisorStrategy.Stop } } diff --git a/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/actors/utils/MessageStatus.scala b/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/actors/utils/MessageStatus.scala index 801cc2bc..f0fdcbdb 100644 --- a/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/actors/utils/MessageStatus.scala +++ b/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/actors/utils/MessageStatus.scala @@ -8,7 +8,7 @@ trait MessageStatus[Key, Value] { sealed abstract class Status extends DeadLetterSuppression { def key: Key } - case class Success(key: Key, result: Value) extends Status + case class Success(key: Key, result: Value) extends Status case class Failure(key: Key, error: Throwable) extends Status def wrapFuture(key: Key, future: Future[Value])(implicit ec: ExecutionContext): Future[this.Status] = { diff --git a/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/actors/utils/PendingOperation.scala b/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/actors/utils/PendingOperation.scala index c3f0aecb..db95733b 100644 --- a/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/actors/utils/PendingOperation.scala +++ b/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/actors/utils/PendingOperation.scala @@ -4,7 +4,6 @@ import akka.actor.ActorRef import scala.collection.mutable - object PendingOperation { def apply[Key <: AnyRef]: PendingOperation[Key] = { new PendingOperation[Key]() @@ -28,10 +27,9 @@ class PendingOperation[Key <: AnyRef] { def removeWaiter(actor: ActorRef): Unit = { subscribers .withFilter(_._2.contains(actor)) - .foreach { - case (key, actors) ⇒ - actors -= actor - if (actors.isEmpty) subscribers -= key + .foreach { case (key, actors) ⇒ + actors -= actor + if (actors.isEmpty) subscribers -= key } } @@ -43,11 +41,10 @@ class PendingOperation[Key <: AnyRef] { subscribers.remove(key).foreach(_.foreach(_ ! result)) } - def finishAll(f: Key => AnyRef)(implicit sender: ActorRef = ActorRef.noSender): Unit = { - subscribers.foreach { - case (key, actors) => - val result = f(key) - actors.foreach(_ ! result) + def finishAll(f: Key ⇒ AnyRef)(implicit sender: ActorRef = ActorRef.noSender): Unit = { + subscribers.foreach { case (key, actors) ⇒ + val result = f(key) + actors.foreach(_ ! result) } subscribers.clear() } diff --git a/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/actors/utils/StringEventBus.scala b/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/actors/utils/StringEventBus.scala index 8a854488..d651a4e4 100644 --- a/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/actors/utils/StringEventBus.scala +++ b/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/actors/utils/StringEventBus.scala @@ -1,15 +1,13 @@ package com.karasiq.shadowcloud.actors.utils - - import akka.actor.ActorRef import akka.event.{ActorEventBus, LookupClassification} final class StringEventBus[E](extract: E ⇒ String) extends ActorEventBus with LookupClassification { - type Event = E + type Event = E type Classifier = String - protected def mapSize(): Int = 16 - protected def classify(event: E): String = extract(event) + protected def mapSize(): Int = 16 + protected def classify(event: E): String = extract(event) protected def publish(event: E, subscriber: ActorRef): Unit = subscriber ! event } diff --git a/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/compression/StreamCompression.scala b/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/compression/StreamCompression.scala index d3f774bf..d98db3e4 100644 --- a/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/compression/StreamCompression.scala +++ b/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/compression/StreamCompression.scala @@ -8,10 +8,10 @@ import com.karasiq.shadowcloud.compression.lz4.LZ4Streams object StreamCompression { object CompressionType extends Enumeration { - val none = Value(0, "none") - val gzip = Value(1, "gzip") + val none = Value(0, "none") + val gzip = Value(1, "gzip") val deflate = Value(2, "deflate") - val lz4 = Value(3, "lz4") + val lz4 = Value(3, "lz4") } def compress(compressionType: CompressionType.Value): Flow[ByteString, ByteString, NotUsed] = { @@ -24,7 +24,7 @@ object StreamCompression { def decompress: Flow[ByteString, ByteString, NotUsed] = { Flow[ByteString].dropWhile(_.isEmpty).prefixAndTail(1).flatMapConcat { case (head +: Nil, stream) ⇒ val dataStream = Source.single(head.drop(1)).concat(stream) - val compType = CompressionType(java.lang.Byte.toUnsignedInt(head.head)) + val compType = CompressionType(java.lang.Byte.toUnsignedInt(head.head)) dataStream.via(rawStreams.decompress(compType)) } } diff --git a/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/compression/lz4/package.scala b/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/compression/lz4/package.scala index 0e8d770c..0c97bc51 100644 --- a/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/compression/lz4/package.scala +++ b/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/compression/lz4/package.scala @@ -12,7 +12,7 @@ import com.karasiq.shadowcloud.utils.ByteStringOutputStream package object lz4 { // TODO: LZ4FrameInputStream, customize compression level - private type LZ4InputStream = net.jpountz.lz4.LZ4BlockInputStream + private type LZ4InputStream = net.jpountz.lz4.LZ4BlockInputStream private type LZ4OutputStream = net.jpountz.lz4.LZ4BlockOutputStream object LZ4Streams { @@ -37,12 +37,12 @@ package object lz4 { } private final class LZ4Compress extends GraphStage[FlowShape[ByteString, ByteString]] { - val inlet = Inlet[ByteString]("LZ4Compress.in") + val inlet = Inlet[ByteString]("LZ4Compress.in") val outlet = Outlet[ByteString]("LZ4Compress.out") - val shape = FlowShape(inlet, outlet) + val shape = FlowShape(inlet, outlet) def createLogic(inheritedAttributes: Attributes) = new GraphStageLogic(shape) with InHandler with OutHandler { - private[this] val bsOutputStream = ByteStringOutputStream() + private[this] val bsOutputStream = ByteStringOutputStream() private[this] val lz4OutputStream = new LZ4OutputStream(bsOutputStream) def onPull(): Unit = { @@ -66,7 +66,7 @@ package object lz4 { override def onUpstreamFinish(): Unit = { lz4OutputStream.close() val lastBlock = bsOutputStream.toByteString - if(lastBlock.nonEmpty) { + if (lastBlock.nonEmpty) { bsOutputStream.clear() emit(outlet, lastBlock, () ⇒ complete(outlet)) } else { diff --git a/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/providers/KeyProvider.scala b/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/providers/KeyProvider.scala index f1ea3a0f..2248b8b7 100644 --- a/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/providers/KeyProvider.scala +++ b/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/providers/KeyProvider.scala @@ -11,4 +11,4 @@ trait KeyProvider { def addKeySet(key: KeySet, regionSet: RegionSet = RegionSet.all, forEncryption: Boolean = true, forDecryption: Boolean = true): Future[KeySet] def modifyKeySet(keyId: KeyId, regionSet: RegionSet = RegionSet.all, forEncryption: Boolean = true, forDecryption: Boolean = true): Future[Done] def getKeyChain(): Future[KeyChain] -} \ No newline at end of file +} diff --git a/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/serialization/SerializationModule.scala b/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/serialization/SerializationModule.scala index 7e2b6eb4..4efcb789 100644 --- a/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/serialization/SerializationModule.scala +++ b/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/serialization/SerializationModule.scala @@ -6,5 +6,5 @@ import akka.util.ByteString trait SerializationModule { def toBytes[T <: AnyRef](value: T): ByteString - def fromBytes[T <: AnyRef : ClassTag](value: ByteString): T + def fromBytes[T <: AnyRef: ClassTag](value: ByteString): T } diff --git a/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/serialization/StreamSerialization.scala b/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/serialization/StreamSerialization.scala index aa0f2de9..c0b28002 100644 --- a/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/serialization/StreamSerialization.scala +++ b/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/serialization/StreamSerialization.scala @@ -1,6 +1,5 @@ package com.karasiq.shadowcloud.serialization - import akka.NotUsed import akka.stream.scaladsl.{Flow, Framing} import akka.util.ByteString @@ -20,8 +19,7 @@ object StreamSerialization { Framing.simpleFramingProtocolDecoder(frameLimit) } - def serializeFramed[T <: AnyRef : ClassTag](serialization: SerializationModule, - frameLimit: Int): Flow[T, ByteString, NotUsed] = { + def serializeFramed[T <: AnyRef: ClassTag](serialization: SerializationModule, frameLimit: Int): Flow[T, ByteString, NotUsed] = { Flow[T] .via(StreamSerialization(serialization).toBytes) // .filter(_.length <= frameLimit) @@ -29,8 +27,7 @@ object StreamSerialization { .named("framedSerialize") } - def deserializeFramed[T <: AnyRef : ClassTag](serialization: SerializationModule, - frameLimit: Int): Flow[ByteString, T, NotUsed] = { + def deserializeFramed[T <: AnyRef: ClassTag](serialization: SerializationModule, frameLimit: Int): Flow[ByteString, T, NotUsed] = { Flow[ByteString] .via(deframe(frameLimit)) .via(StreamSerialization(serialization).fromBytes[T]) @@ -45,7 +42,7 @@ final class StreamSerialization(private val module: SerializationModule) extends .named("toBytes") } - def fromBytes[T <: AnyRef : ClassTag]: Flow[ByteString, T, NotUsed] = { + def fromBytes[T <: AnyRef: ClassTag]: Flow[ByteString, T, NotUsed] = { Flow[ByteString] .map(module.fromBytes[T]) .named("fromBytes") diff --git a/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/serialization/internal/AkkaSerializationExtensionModule.scala b/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/serialization/internal/AkkaSerializationExtensionModule.scala index 35c56c5d..7c933950 100644 --- a/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/serialization/internal/AkkaSerializationExtensionModule.scala +++ b/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/serialization/internal/AkkaSerializationExtensionModule.scala @@ -1,7 +1,6 @@ package com.karasiq.shadowcloud.serialization.internal - -import akka.serialization.{Serialization => AkkaSerialization} +import akka.serialization.{Serialization ⇒ AkkaSerialization} import akka.util.ByteString import com.karasiq.shadowcloud.serialization.SerializationModule import com.karasiq.shadowcloud.utils.ByteStringUnsafe @@ -13,7 +12,7 @@ private[serialization] final class AkkaSerializationExtensionModule(extension: A extension.serialize(value).map(ByteString.fromArrayUnsafe).get } - def fromBytes[T <: AnyRef : ClassTag](value: ByteString): T = { + def fromBytes[T <: AnyRef: ClassTag](value: ByteString): T = { extension.deserialize(ByteStringUnsafe.getArray(value), implicitly[ClassTag[T]].runtimeClass.asInstanceOf[Class[T]]).get } } diff --git a/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/serialization/internal/AkkaSerializerModule.scala b/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/serialization/internal/AkkaSerializerModule.scala index d9e568e3..da7b6ebe 100644 --- a/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/serialization/internal/AkkaSerializerModule.scala +++ b/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/serialization/internal/AkkaSerializerModule.scala @@ -1,7 +1,6 @@ package com.karasiq.shadowcloud.serialization.internal - -import akka.serialization.{Serializer => AkkaSerializer} +import akka.serialization.{Serializer ⇒ AkkaSerializer} import akka.util.ByteString import com.karasiq.shadowcloud.serialization.SerializationModule import com.karasiq.shadowcloud.utils.ByteStringUnsafe @@ -13,7 +12,7 @@ private[serialization] final class AkkaSerializerModule(serializer: AkkaSerializ ByteString.fromArrayUnsafe(serializer.toBinary(value)) } - def fromBytes[T <: AnyRef : ClassTag](value: ByteString): T = { + def fromBytes[T <: AnyRef: ClassTag](value: ByteString): T = { serializer.fromBinary(ByteStringUnsafe.getArray(value), Some(implicitly[ClassTag[T]].runtimeClass)).asInstanceOf[T] } } diff --git a/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/serialization/internal/CompanionReflectSerializer.scala b/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/serialization/internal/CompanionReflectSerializer.scala index 5fe3f886..bccb2f3d 100644 --- a/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/serialization/internal/CompanionReflectSerializer.scala +++ b/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/serialization/internal/CompanionReflectSerializer.scala @@ -4,12 +4,11 @@ import scala.collection.concurrent.TrieMap private[serialization] trait CompanionReflectSerializer { protected type Companion - private[this] val runtime = scala.reflect.runtime.universe.runtimeMirror(getClass.getClassLoader) + private[this] val runtime = scala.reflect.runtime.universe.runtimeMirror(getClass.getClassLoader) private[this] val companionMap = TrieMap.empty[Class[_], Any] private[this] def getCompanionWithReflection[C](`type`: Class[_]): C = { - runtime.reflectModule(runtime.classSymbol(`type`).companion.asModule) - .instance.asInstanceOf[C] + runtime.reflectModule(runtime.classSymbol(`type`).companion.asModule).instance.asInstanceOf[C] } protected def getCompanion[C](`type`: Class[_]): C = { diff --git a/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/storage/utils/ChunkKeyMapper.scala b/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/storage/utils/ChunkKeyMapper.scala index dac37fce..0a16f298 100644 --- a/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/storage/utils/ChunkKeyMapper.scala +++ b/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/storage/utils/ChunkKeyMapper.scala @@ -1,7 +1,5 @@ package com.karasiq.shadowcloud.storage.utils - - import akka.util.ByteString import com.karasiq.common.configs.ConfigImplicits._ import com.karasiq.shadowcloud.model.{Chunk, ChunkId} @@ -13,15 +11,15 @@ private[shadowcloud] trait ChunkKeyMapper extends (Chunk ⇒ ByteString) { } private[shadowcloud] object ChunkKeyMapper { - val hash: ChunkKeyMapper = (chunk: Chunk) ⇒ chunk.checksum.hash + val hash: ChunkKeyMapper = (chunk: Chunk) ⇒ chunk.checksum.hash val encryptedHash: ChunkKeyMapper = (chunk: Chunk) ⇒ chunk.checksum.encHash - val doubleHash: ChunkKeyMapper = (chunk: Chunk) ⇒ chunk.checksum.hash ++ chunk.checksum.encHash + val doubleHash: ChunkKeyMapper = (chunk: Chunk) ⇒ chunk.checksum.hash ++ chunk.checksum.encHash def forName(name: String, config: Config = Utils.emptyConfig): ChunkKeyMapper = name match { // Predefined - case "hash" ⇒ hash + case "hash" ⇒ hash case "encrypted-hash" ⇒ encryptedHash - case "double-hash" ⇒ doubleHash + case "double-hash" ⇒ doubleHash // Custom class case str ⇒ diff --git a/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/storage/utils/mappers/CompositeKeyMapper.scala b/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/storage/utils/mappers/CompositeKeyMapper.scala index 1bd592c5..61d09a45 100644 --- a/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/storage/utils/mappers/CompositeKeyMapper.scala +++ b/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/storage/utils/mappers/CompositeKeyMapper.scala @@ -17,7 +17,7 @@ private object CompositeKeyMapper { case object XOR extends Strategy { override def apply(v1: ChunkId, v2: ChunkId): ChunkId = { - val bsb = ByteString.newBuilder + val bsb = ByteString.newBuilder val (data, key) = if (v1.length >= v2.length) (v1, v2) else (v2, v1) bsb.sizeHint(data.length) @@ -30,8 +30,8 @@ private object CompositeKeyMapper { def forName(str: String): Strategy = str.toLowerCase match { case "concat" ⇒ Concat - case "xor" ⇒ XOR - case _ ⇒ throw new IllegalArgumentException(str) + case "xor" ⇒ XOR + case _ ⇒ throw new IllegalArgumentException(str) } } } @@ -41,7 +41,7 @@ private[shadowcloud] class CompositeKeyMapper(config: Config) extends ChunkKeyMa private[this] object settings extends ConfigImplicits { val strategy = Strategy.forName(config.withDefault("concat", _.getString("strategy"))) - val mappers = config.getStrings("mappers") + val mappers = config.getStrings("mappers") } require(settings.mappers.nonEmpty, "No mappers specified") diff --git a/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/utils/ByteStringUnsafe.scala b/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/utils/ByteStringUnsafe.scala index 9cc8a4ae..40994f4b 100644 --- a/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/utils/ByteStringUnsafe.scala +++ b/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/utils/ByteStringUnsafe.scala @@ -6,7 +6,7 @@ import scala.util.Try object ByteStringUnsafe { private[this] val ByteString1CRef = Try(Class.forName("akka.util.ByteString$ByteString1C")) - private[this] val BytesFieldRef = ByteString1CRef.map { cls => + private[this] val BytesFieldRef = ByteString1CRef.map { cls ⇒ val f = cls.getDeclaredField("bytes") f.setAccessible(true) f @@ -15,8 +15,8 @@ object ByteStringUnsafe { def getArray(bs: ByteString): Array[Byte] = if (BytesFieldRef.isSuccess && ByteString1CRef.get.isInstance(bs)) { BytesFieldRef.get.get(bs) match { - case bs: Array[Byte] => bs - case _ => bs.toArray // Patched version + case bs: Array[Byte] ⇒ bs + case _ ⇒ bs.toArray // Patched version } } else bs.toArray diff --git a/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/utils/FileSystemUtils.scala b/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/utils/FileSystemUtils.scala index 91d7d59d..40e29a70 100644 --- a/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/utils/FileSystemUtils.scala +++ b/utils/.jvm/src/main/scala/com/karasiq/shadowcloud/utils/FileSystemUtils.scala @@ -9,45 +9,53 @@ import akka.stream.scaladsl.Source import akka.stream.{ActorAttributes, Attributes} import scala.collection.JavaConverters._ -import scala.collection.mutable.{Set => MSet} +import scala.collection.mutable.{Set ⇒ MSet} private[shadowcloud] object FileSystemUtils { private[this] val forbiddenChars = """[<>:"/\\|?*]""".r - def walkFileTree(folder: Path, - walkUntil: Path ⇒ Boolean = _ ⇒ true, - includeFiles: Boolean = true, - includeDirs: Boolean = true, - maxDepth: Int = Int.MaxValue, - options: Set[FileVisitOption] = Set.empty): Source[Path, NotUsed] = { - Source.single(folder) + def walkFileTree( + folder: Path, + walkUntil: Path ⇒ Boolean = _ ⇒ true, + includeFiles: Boolean = true, + includeDirs: Boolean = true, + maxDepth: Int = Int.MaxValue, + options: Set[FileVisitOption] = Set.empty + ): Source[Path, NotUsed] = { + Source + .single(folder) .mapConcat { path ⇒ val builder = List.newBuilder[Path] val visited = MSet.empty[Path] - Files.walkFileTree(path, options.asJava, maxDepth, new SimpleFileVisitor[Path] { - override def visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult = { - if (includeFiles) builder += file - FileVisitResult.CONTINUE - } + Files.walkFileTree( + path, + options.asJava, + maxDepth, + new SimpleFileVisitor[Path] { + override def visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult = { + if (includeFiles) builder += file + FileVisitResult.CONTINUE + } - override def preVisitDirectory(dir: Path, attrs: BasicFileAttributes): FileVisitResult = { - if (visited.contains(dir)) { - FileVisitResult.SKIP_SUBTREE - } else { - visited += dir - if (includeDirs) builder += dir - if (walkUntil(dir)) FileVisitResult.CONTINUE else FileVisitResult.SKIP_SUBTREE + override def preVisitDirectory(dir: Path, attrs: BasicFileAttributes): FileVisitResult = { + if (visited.contains(dir)) { + FileVisitResult.SKIP_SUBTREE + } else { + visited += dir + if (includeDirs) builder += dir + if (walkUntil(dir)) FileVisitResult.CONTINUE else FileVisitResult.SKIP_SUBTREE + } } - } - override def postVisitDirectory(dir: Path, exc: IOException): FileVisitResult = { - FileVisitResult.CONTINUE - } + override def postVisitDirectory(dir: Path, exc: IOException): FileVisitResult = { + FileVisitResult.CONTINUE + } - override def visitFileFailed(file: Path, exc: IOException): FileVisitResult = { - FileVisitResult.CONTINUE + override def visitFileFailed(file: Path, exc: IOException): FileVisitResult = { + FileVisitResult.CONTINUE + } } - }) + ) builder.result() } .withAttributes(Attributes.name("walkFileTree") and ActorAttributes.IODispatcher) @@ -60,16 +68,19 @@ private[shadowcloud] object FileSystemUtils { def getFolderSize(folder: Path): Long = concurrent.blocking { if (!Files.exists(folder)) return 0L var result = 0L - Files.walkFileTree(folder, new SimpleFileVisitor[Path] { - override def visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult = { - result += Files.size(file) - FileVisitResult.CONTINUE - } + Files.walkFileTree( + folder, + new SimpleFileVisitor[Path] { + override def visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult = { + result += Files.size(file) + FileVisitResult.CONTINUE + } - override def preVisitDirectory(dir: Path, attrs: BasicFileAttributes): FileVisitResult = { - if (attrs.isSymbolicLink) FileVisitResult.SKIP_SUBTREE else FileVisitResult.CONTINUE + override def preVisitDirectory(dir: Path, attrs: BasicFileAttributes): FileVisitResult = { + if (attrs.isSymbolicLink) FileVisitResult.SKIP_SUBTREE else FileVisitResult.CONTINUE + } } - }) + ) result } diff --git a/utils/src/main/scala/com/karasiq/shadowcloud/storage/utils/IndexMerger.scala b/utils/src/main/scala/com/karasiq/shadowcloud/storage/utils/IndexMerger.scala index cc4888f1..d93566ff 100644 --- a/utils/src/main/scala/com/karasiq/shadowcloud/storage/utils/IndexMerger.scala +++ b/utils/src/main/scala/com/karasiq/shadowcloud/storage/utils/IndexMerger.scala @@ -54,15 +54,13 @@ private[shadowcloud] object IndexMerger { restore(createState(index)) } - /** - * Sequential index merger + /** Sequential index merger */ def sequential(): IndexMerger[SequenceNr] = { this.apply[SequenceNr] } - /** - * Multi-sequence index merger + /** Multi-sequence index merger */ def region(): IndexMerger[RegionKey] = { this.apply[RegionKey] @@ -77,12 +75,12 @@ private[shadowcloud] object IndexMerger { this.restore(filteredState) case IndexScope.UntilSequenceNr(sequenceNr) ⇒ - val state = this.createState(index) + val state = this.createState(index) val filteredState = state.copy(diffs = state.diffs.filter(kv ⇒ ko.sequenceNr(kv._1) <= sequenceNr), pending = IndexDiff.empty) this.restore(filteredState) case IndexScope.UntilTime(timestamp) ⇒ - val state = this.createState(index) + val state = this.createState(index) val filteredState = state.copy(diffs = state.diffs.filter(kv ⇒ ko.timestamp(kv._1) <= timestamp), pending = IndexDiff.empty) this.restore(filteredState) } @@ -113,7 +111,7 @@ private[shadowcloud] object IndexMerger { object RegionKey { implicit val ordering: Ordering[RegionKey] = Ordering.by(key ⇒ (key.timestamp, key.storageId, key.sequenceNr)) - val zero: RegionKey = RegionKey() + val zero: RegionKey = RegionKey() } @SerialVersionUID(0L) @@ -133,16 +131,16 @@ private[shadowcloud] object IndexMerger { object IndexKeyOrdering { implicit val sequenceNrOrdering: IndexKeyOrdering[SequenceNr] = new IndexKeyOrdering[SequenceNr] { implicit val ordering: Ordering[SequenceNr] = Ordering.Long - def zero: SequenceNr = SequenceNr.zero + def zero: SequenceNr = SequenceNr.zero def sequenceNr(key: SequenceNr): SequenceNr = key - def timestamp(key: SequenceNr): SequenceNr = key + def timestamp(key: SequenceNr): SequenceNr = key } implicit val regionKeyOrdering: IndexKeyOrdering[RegionKey] = new IndexKeyOrdering[RegionKey] { implicit def ordering: Ordering[RegionKey] = RegionKey.ordering - def zero = RegionKey.zero - def sequenceNr(key: RegionKey) = key.sequenceNr - def timestamp(key: RegionKey) = key.timestamp + def zero = RegionKey.zero + def sequenceNr(key: RegionKey) = key.sequenceNr + def timestamp(key: RegionKey) = key.timestamp } } } diff --git a/utils/src/main/scala/com/karasiq/shadowcloud/storage/utils/internal/DefaultIndexMerger.scala b/utils/src/main/scala/com/karasiq/shadowcloud/storage/utils/internal/DefaultIndexMerger.scala index 044f3774..e0f53129 100644 --- a/utils/src/main/scala/com/karasiq/shadowcloud/storage/utils/internal/DefaultIndexMerger.scala +++ b/utils/src/main/scala/com/karasiq/shadowcloud/storage/utils/internal/DefaultIndexMerger.scala @@ -10,23 +10,23 @@ import com.karasiq.shadowcloud.utils.MergeUtil.Decider import scala.collection.{SortedMap, mutable} private[storage] final class DefaultIndexMerger[@specialized(Long) T](firstKey: T)(implicit ord: Ordering[T]) extends IndexMerger[T] { - private[this] var _diffs = mutable.SortedMap.empty[T, IndexDiff] - private[this] var _chunks = ChunkIndex.empty + private[this] var _diffs = mutable.SortedMap.empty[T, IndexDiff] + private[this] var _chunks = ChunkIndex.empty private[this] var _folders = FolderIndex.empty - private[this] var _merged = IndexDiff.empty + private[this] var _merged = IndexDiff.empty private[this] var _pending = IndexDiff.empty - def lastSequenceNr: T = if (_diffs.isEmpty) firstKey else _diffs.lastKey - def chunks: ChunkIndex = _chunks - def folders: FolderIndex = _folders + def lastSequenceNr: T = if (_diffs.isEmpty) firstKey else _diffs.lastKey + def chunks: ChunkIndex = _chunks + def folders: FolderIndex = _folders def diffs: SortedMap[T, IndexDiff] = _diffs - def mergedDiff: IndexDiff = _merged - def pending: IndexDiff = _pending + def mergedDiff: IndexDiff = _merged + def pending: IndexDiff = _pending def add(sequenceNr: T, diff: IndexDiff): Unit = { _diffs.get(sequenceNr) match { case Some(`diff`) ⇒ - // Pass + // Pass case Some(existing) ⇒ throw SCExceptions.DiffConflict(existing, diff) @@ -42,7 +42,7 @@ private[storage] final class DefaultIndexMerger[@specialized(Long) T](firstKey: deletePending(diff) } } - + def delete(sequenceNrs: Set[T]): Unit = { _diffs --= sequenceNrs rebuildIndex() diff --git a/utils/src/main/scala/com/karasiq/shadowcloud/utils/DiffStats.scala b/utils/src/main/scala/com/karasiq/shadowcloud/utils/DiffStats.scala index 8a8fde8a..a29f6f01 100644 --- a/utils/src/main/scala/com/karasiq/shadowcloud/utils/DiffStats.scala +++ b/utils/src/main/scala/com/karasiq/shadowcloud/utils/DiffStats.scala @@ -1,22 +1,39 @@ package com.karasiq.shadowcloud.utils - - import com.karasiq.common.memory.MemorySize import com.karasiq.shadowcloud.index.diffs.IndexDiff import com.karasiq.shadowcloud.index.utils.HasEmpty import com.karasiq.shadowcloud.storage.utils.IndexMerger -case class DiffStats(diffs: Int, files: Int, folders: Int, chunks: Int, - fileDeletes: Int, folderDeletes: Int, chunkDeletes: Int, - createdBytes: Long, deletedBytes: Long) extends HasEmpty { +case class DiffStats( + diffs: Int, + files: Int, + folders: Int, + chunks: Int, + fileDeletes: Int, + folderDeletes: Int, + chunkDeletes: Int, + createdBytes: Long, + deletedBytes: Long +) extends HasEmpty { - require(diffs >= 0 && files >= 0 && folders >= 0 && chunks >= 0 && - fileDeletes >= 0 && folderDeletes >= 0 && chunkDeletes >= 0 && createdBytes >= 0 && deletedBytes >= 0) + require( + diffs >= 0 && files >= 0 && folders >= 0 && chunks >= 0 && + fileDeletes >= 0 && folderDeletes >= 0 && chunkDeletes >= 0 && createdBytes >= 0 && deletedBytes >= 0 + ) def +(stat: DiffStats): DiffStats = { - DiffStats(diffs + stat.diffs, files + stat.files, folders + stat.folders, chunks + stat.chunks, fileDeletes + stat.fileDeletes, - folderDeletes + stat.folderDeletes, chunkDeletes + stat.chunkDeletes, createdBytes + stat.createdBytes, deletedBytes + stat.deletedBytes) + DiffStats( + diffs + stat.diffs, + files + stat.files, + folders + stat.folders, + chunks + stat.chunks, + fileDeletes + stat.fileDeletes, + folderDeletes + stat.folderDeletes, + chunkDeletes + stat.chunkDeletes, + createdBytes + stat.createdBytes, + deletedBytes + stat.deletedBytes + ) } def +(diff: IndexDiff): DiffStats = { @@ -46,7 +63,7 @@ object DiffStats { def apply(diffs: IndexDiff*): DiffStats = { if (diffs.isEmpty) return empty var diffCount, files, folders, chunks, fileDeletes, folderDeletes, chunkDeletes = 0 - var createdBytes, deletedBytes = 0L + var createdBytes, deletedBytes = 0L diffs.foreach { diff ⇒ diffCount += 1 diff.folders.folders.foreach { fd ⇒ diff --git a/utils/src/main/scala/com/karasiq/shadowcloud/utils/ProviderInstantiator.scala b/utils/src/main/scala/com/karasiq/shadowcloud/utils/ProviderInstantiator.scala index 7c35edd3..35980beb 100644 --- a/utils/src/main/scala/com/karasiq/shadowcloud/utils/ProviderInstantiator.scala +++ b/utils/src/main/scala/com/karasiq/shadowcloud/utils/ProviderInstantiator.scala @@ -19,4 +19,4 @@ private[shadowcloud] object ProviderInstantiator { def fromConfig[T](config: Config): T = { withConfig[T](config.getString("class"), config) } -} \ No newline at end of file +} diff --git a/utils/test/.jvm/src/main/scala/com/karasiq/shadowcloud/test/crypto/utils/CryptoTestVectors.scala b/utils/test/.jvm/src/main/scala/com/karasiq/shadowcloud/test/crypto/utils/CryptoTestVectors.scala index fbdcc5df..8c59fba8 100644 --- a/utils/test/.jvm/src/main/scala/com/karasiq/shadowcloud/test/crypto/utils/CryptoTestVectors.scala +++ b/utils/test/.jvm/src/main/scala/com/karasiq/shadowcloud/test/crypto/utils/CryptoTestVectors.scala @@ -44,7 +44,7 @@ class CryptoTestVectors(testVectorsFolder: FSPath) { } private[this] def writeTestVector(parameters: EncryptionParameters, plain: ByteString, encrypted: ByteString): ByteString = { - val bsOutput = ByteStringOutputStream() + val bsOutput = ByteStringOutputStream() val objOutput = new ObjectOutputStream(bsOutput) objOutput.writeObject(parameters) objOutput.writeObject(plain) @@ -55,11 +55,11 @@ class CryptoTestVectors(testVectorsFolder: FSPath) { } private[this] def readTestVector(str: ByteString): (EncryptionParameters, ByteString, ByteString) = { - val bsInput = ByteStringInputStream(str) - val objInput = new ObjectInputStream(bsInput) + val bsInput = ByteStringInputStream(str) + val objInput = new ObjectInputStream(bsInput) val parameters = objInput.readObject().asInstanceOf[EncryptionParameters] - val plain = objInput.readObject().asInstanceOf[ByteString] - val encrypted = objInput.readObject().asInstanceOf[ByteString] + val plain = objInput.readObject().asInstanceOf[ByteString] + val encrypted = objInput.readObject().asInstanceOf[ByteString] objInput.close() (parameters, plain, encrypted) } @@ -70,4 +70,4 @@ object CryptoTestVectors { val path = Paths.get(s"./crypto/$name/src/test/resources/test-vectors") new CryptoTestVectors(path) } -} \ No newline at end of file +} diff --git a/utils/test/.jvm/src/main/scala/com/karasiq/shadowcloud/test/utils/ActorSpec.scala b/utils/test/.jvm/src/main/scala/com/karasiq/shadowcloud/test/utils/ActorSpec.scala index 9be8141f..125a85a1 100644 --- a/utils/test/.jvm/src/main/scala/com/karasiq/shadowcloud/test/utils/ActorSpec.scala +++ b/utils/test/.jvm/src/main/scala/com/karasiq/shadowcloud/test/utils/ActorSpec.scala @@ -7,8 +7,7 @@ import org.scalatest.{BeforeAndAfterAll, Matchers, Suite} import scala.concurrent.duration._ -abstract class ActorSpec extends TestKit(ActorSystem("test")) with ImplicitSender - with Suite with Matchers with ScalaFutures with BeforeAndAfterAll { +abstract class ActorSpec extends TestKit(ActorSystem("test")) with ImplicitSender with Suite with Matchers with ScalaFutures with BeforeAndAfterAll { override implicit def patienceConfig: PatienceConfig = { PatienceConfig(10 seconds, 100 millis) @@ -19,4 +18,3 @@ abstract class ActorSpec extends TestKit(ActorSystem("test")) with ImplicitSende super.afterAll() } } - diff --git a/utils/test/src/main/scala/com/karasiq/shadowcloud/test/utils/ByteStringImplicits.scala b/utils/test/src/main/scala/com/karasiq/shadowcloud/test/utils/ByteStringImplicits.scala index 462b8e90..cc4b9c47 100644 --- a/utils/test/src/main/scala/com/karasiq/shadowcloud/test/utils/ByteStringImplicits.scala +++ b/utils/test/src/main/scala/com/karasiq/shadowcloud/test/utils/ByteStringImplicits.scala @@ -1,7 +1,5 @@ package com.karasiq.shadowcloud.test.utils - - import akka.util.ByteString import com.karasiq.common.encoding.HexString import com.karasiq.shadowcloud.model.Chunk diff --git a/utils/test/src/main/scala/com/karasiq/shadowcloud/test/utils/TestUtils.scala b/utils/test/src/main/scala/com/karasiq/shadowcloud/test/utils/TestUtils.scala index 18941cbc..305da1ca 100644 --- a/utils/test/src/main/scala/com/karasiq/shadowcloud/test/utils/TestUtils.scala +++ b/utils/test/src/main/scala/com/karasiq/shadowcloud/test/utils/TestUtils.scala @@ -28,18 +28,52 @@ object TestUtils { } def indexedBytes: (ByteString, File) = { - val text = ByteString("""You may have noticed various code patterns that emerge when testing stream pipelines. Akka Stream has a separate akka-stream-testkit module that provides tools specifically for writing stream tests. This module comes with two main components that are TestSource and TestSink which provide sources and sinks that materialize to probes that allow fluent API.""") + val text = ByteString( + """You may have noticed various code patterns that emerge when testing stream pipelines. Akka Stream has a separate akka-stream-testkit module that provides tools specifically for writing stream tests. This module comes with two main components that are TestSource and TestSink which provide sources and sinks that materialize to probes that allow fluent API.""" + ) val hashingMethod = HashingMethod("SHA1") - val textHash = HexString.decode("2f5a0c419cfeb92f05888ae3468e54fee3ee1726") - val preCalcHashes = Vector("f660847d03634f41c45f7be337b02973a083721a", "dfa6cbe4eb725d390e3339075fe420791a5a394f", "e63bf72054623e911ce6a995dc520527d7fe2e2d", "802f6e7f54ca13c650741e65f188b0bdb023cb15").map(HexString.decode) + val textHash = HexString.decode("2f5a0c419cfeb92f05888ae3468e54fee3ee1726") + val preCalcHashes = Vector( + "f660847d03634f41c45f7be337b02973a083721a", + "dfa6cbe4eb725d390e3339075fe420791a5a394f", + "e63bf72054623e911ce6a995dc520527d7fe2e2d", + "802f6e7f54ca13c650741e65f188b0bdb023cb15" + ).map(HexString.decode) val chunks = Seq( - Chunk(Checksum(hashingMethod, hashingMethod, 100, preCalcHashes(0), 100, preCalcHashes(0)), EncryptionParameters.empty, Data(text.slice(0, 100), text.slice(0, 100))), - Chunk(Checksum(hashingMethod, hashingMethod, 100, preCalcHashes(1), 100, preCalcHashes(1)), EncryptionParameters.empty, Data(text.slice(100, 200), text.slice(100, 200))), - Chunk(Checksum(hashingMethod, hashingMethod, 100, preCalcHashes(2), 100, preCalcHashes(2)), EncryptionParameters.empty, Data(text.slice(200, 300), text.slice(200, 300))), - Chunk(Checksum(hashingMethod, hashingMethod, 56, preCalcHashes(3), 56, preCalcHashes(3)), EncryptionParameters.empty, Data(text.slice(300, 356), text.slice(300, 356))) + Chunk( + Checksum(hashingMethod, hashingMethod, 100, preCalcHashes(0), 100, preCalcHashes(0)), + EncryptionParameters.empty, + Data(text.slice(0, 100), text.slice(0, 100)) + ), + Chunk( + Checksum(hashingMethod, hashingMethod, 100, preCalcHashes(1), 100, preCalcHashes(1)), + EncryptionParameters.empty, + Data(text.slice(100, 200), text.slice(100, 200)) + ), + Chunk( + Checksum(hashingMethod, hashingMethod, 100, preCalcHashes(2), 100, preCalcHashes(2)), + EncryptionParameters.empty, + Data(text.slice(200, 300), text.slice(200, 300)) + ), + Chunk( + Checksum(hashingMethod, hashingMethod, 56, preCalcHashes(3), 56, preCalcHashes(3)), + EncryptionParameters.empty, + Data(text.slice(300, 356), text.slice(300, 356)) + ) + ) + ( + text, + File( + Path.root / "test.txt", + UUID.fromString("c3cd9085-b2d5-43fb-b85c-a8448698f7a6"), + 0, + Timestamp(testTimestamp, testTimestamp), + SerializedProps.empty, + Checksum(hashingMethod, hashingMethod, 356, textHash, 356, textHash), + chunks + ) ) - (text, File(Path.root / "test.txt", UUID.fromString("c3cd9085-b2d5-43fb-b85c-a8448698f7a6"), 0, Timestamp(testTimestamp, testTimestamp), SerializedProps.empty, Checksum(hashingMethod, hashingMethod, 356, textHash, 356, textHash), chunks)) } def testChunk: Chunk = {