Mercurial > hg > monetdb-java
diff tests/OnClientTester.java @ 616:65641a7cea31
Implement line ending conversion for downloads
MonetConnection.Download#getStream returns an InputStream which
converts line endings when in text mode.
The default line ending is the platform line ending but that can be
changed. Setting it to \n can be a useful optimization if you don't
need the \r's anyway.
author | Joeri van Ruth <joeri.van.ruth@monetdbsolutions.com> |
---|---|
date | Wed, 19 Jan 2022 14:58:01 +0100 (2022-01-19) |
parents | 34a15cd8cfc2 |
children | 21d0f4a43697 |
line wrap: on
line diff
--- a/tests/OnClientTester.java +++ b/tests/OnClientTester.java @@ -28,6 +28,7 @@ import java.sql.Statement; import java.sql.SQLException; import java.util.ArrayList; import java.util.List; +import java.util.Random; import java.util.zip.GZIPOutputStream; import static java.nio.file.StandardOpenOption.CREATE_NEW; @@ -147,6 +148,8 @@ public final class OnClientTester { test_LargeUpload(); if (isSelected("LargeDownload")) test_LargeDownload(); + if (isSelected("DownloadCrLf")) + test_DownloadCrLf(); if (isSelected("UploadFromStream")) test_UploadFromStream(); if (isSelected("UploadFromReader")) @@ -602,6 +605,61 @@ public final class OnClientTester { exitTest(); } + private void test_DownloadCrLf() throws SQLException, Failure, IOException { + + // This tests forces line ending conversion and reads in small batches, hoping to trigger corner cases + + initTest("test_DownloadCrLf"); + prepare(); + update("ALTER TABLE foo DROP COLUMN t"); + update("ALTER TABLE foo ADD COLUMN j INT"); + update("INSERT INTO foo SELECT rand() % CASE WHEN value % 10 = 0 THEN 1000 ELSE 10 END AS i, 0 AS j FROM generate_series(0, 500000)"); + ByteArrayOutputStream target = new ByteArrayOutputStream(); + Random rng = new Random(42); + DownloadHandler handler = (handle, name, textMode) -> { + handle.setLineSeparator("\r\n"); + InputStream s = handle.getStream(); + byte[] buf = new byte[10]; + boolean expectEof = false; + for (;;) { + int n = rng.nextInt(buf.length - 1) + 1; + int nread = s.read(buf, 0, n); + if (nread < 0) { + break; + } + target.write(buf, 0, nread); + } + + }; + conn.setDownloadHandler(handler); + update("COPY SELECT * FROM foo INTO 'banana' ON CLIENT"); + // go to String instead of byte[] because Strings have handy replace methods. + String result = new String(target.toByteArray(), StandardCharsets.UTF_8); + + // It should contain only \r\n's, no lonely \r's or \n's. + String replaced = result.replaceAll("\r\n", "XX"); + + assertEq("Index of first lonely \\r", -1, replaced.indexOf('\r')); + assertEq("Index of first lonely \\n", -1, replaced.indexOf('\n')); + + String withoutData = result.replaceAll("[0-9]", ""); + Files.writeString(Path.of("/tmp/x.csv"), withoutData, StandardCharsets.UTF_8); + assertEq("Length after dropping data, modulo 3", 0, withoutData.length() % 3); + for (int i = 0; i < withoutData.length(); i += 3) { + String sub = withoutData.substring(i, i+3); + if (!sub.equals("|\r\n")) { + fail(String.format( + "At index %d out of %d in the skeleton (=digits removed) we find <%02x %02x %02x> instead of <7c 0d 0a>", + i, withoutData.length(), + (int)sub.charAt(0), (int)sub.charAt(1), (int)sub.charAt(2))); + } + } + // only to show some succesful output if the above succeeds + assertEq("Every 3-byte normalized chunk", "|\\r\\n", "|\\r\\n"); + + exitTest(); + } + private void test_UploadFromStream() throws SQLException, Failure { initTest("test_UploadFromStream"); prepare(); @@ -894,6 +952,11 @@ public final class OnClientTester { /* utility methods */ + private void say(String message) throws Failure { + outBuffer.append(message).append("\n"); + throw new Failure(message); + } + private void fail(String message) throws Failure { outBuffer.append("FAILURE: ").append(message).append("\n"); throw new Failure(message);