diff --git a/app/src/androidTest/java/com/maxistar/textpad/test/BasicActivityTest.java b/app/src/androidTest/java/com/maxistar/textpad/test/BasicActivityTest.java index f12ec7f..3bc9960 100644 --- a/app/src/androidTest/java/com/maxistar/textpad/test/BasicActivityTest.java +++ b/app/src/androidTest/java/com/maxistar/textpad/test/BasicActivityTest.java @@ -5,6 +5,7 @@ import static androidx.test.espresso.matcher.ViewMatchers.isDisplayed; import static androidx.test.espresso.matcher.ViewMatchers.withId; +import android.Manifest; import android.content.Context; import android.view.View; import android.widget.TextView; @@ -15,6 +16,7 @@ import androidx.test.platform.app.InstrumentationRegistry; import androidx.test.ext.junit.runners.AndroidJUnit4; import androidx.test.espresso.intent.rule.IntentsTestRule; +import androidx.test.rule.GrantPermissionRule; import org.junit.Rule; import org.junit.Test; @@ -35,7 +37,12 @@ public class BasicActivityTest { - @Rule + @Rule(order = 1) + public GrantPermissionRule mRuntimePermissionRule = GrantPermissionRule.grant( + Manifest.permission.WRITE_EXTERNAL_STORAGE, + Manifest.permission.READ_EXTERNAL_STORAGE); + + @Rule(order = 2) public IntentsTestRule intentsTestRule = new IntentsTestRule<>(EditorActivity.class); diff --git a/app/src/androidTest/java/com/maxistar/textpad/test/EditorExternalChangeTest.java b/app/src/androidTest/java/com/maxistar/textpad/test/EditorExternalChangeTest.java index 410e939..2ecbfae 100644 --- a/app/src/androidTest/java/com/maxistar/textpad/test/EditorExternalChangeTest.java +++ b/app/src/androidTest/java/com/maxistar/textpad/test/EditorExternalChangeTest.java @@ -13,6 +13,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; +import android.Manifest; import android.content.ContentValues; import android.app.Activity; import android.app.Instrumentation.ActivityResult; @@ -29,6 +30,7 @@ import androidx.test.core.app.ApplicationProvider; import androidx.test.espresso.intent.Intents; import androidx.test.ext.junit.runners.AndroidJUnit4; +import androidx.test.rule.GrantPermissionRule; import com.maxistar.textpad.R; import com.maxistar.textpad.ServiceLocator; @@ -42,6 +44,7 @@ import org.junit.After; import org.junit.Assume; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; @@ -57,6 +60,11 @@ public class EditorExternalChangeTest { private Context context; + @Rule + public GrantPermissionRule mRuntimePermissionRule = GrantPermissionRule.grant( + Manifest.permission.WRITE_EXTERNAL_STORAGE, + Manifest.permission.READ_EXTERNAL_STORAGE); + @Before public void setUp() { context = ApplicationProvider.getApplicationContext(); diff --git a/app/src/androidTest/java/com/maxistar/textpad/test/EditorFileEncodingTest.java b/app/src/androidTest/java/com/maxistar/textpad/test/EditorFileEncodingTest.java new file mode 100644 index 0000000..0ebebf4 --- /dev/null +++ b/app/src/androidTest/java/com/maxistar/textpad/test/EditorFileEncodingTest.java @@ -0,0 +1,211 @@ +package com.maxistar.textpad.test; + +import static androidx.test.espresso.Espresso.onView; +import static androidx.test.espresso.assertion.ViewAssertions.matches; +import static androidx.test.espresso.matcher.ViewMatchers.withId; +import static androidx.test.espresso.matcher.ViewMatchers.withText; +import static org.junit.Assert.assertArrayEquals; + +import android.Manifest; +import android.content.ContentValues; +import android.content.Context; +import android.content.Intent; +import android.net.Uri; +import android.os.Build; +import android.preference.PreferenceManager; +import android.provider.MediaStore; +import android.widget.EditText; + +import androidx.test.core.app.ActivityScenario; +import androidx.test.core.app.ApplicationProvider; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import androidx.test.rule.GrantPermissionRule; + +import com.maxistar.textpad.R; +import com.maxistar.textpad.ServiceLocator; +import com.maxistar.textpad.activities.EditorActivity; +import com.maxistar.textpad.recovery.RecoveryRepository; +import com.maxistar.textpad.service.SettingsService; + +import org.junit.After; +import org.junit.Assume; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.InputStream; +import java.lang.reflect.Method; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; + +/** + * Opens documents written in different encodings and checks the editor shows + * the correct text and saves the file back in its original encoding with the + * original byte order mark. + */ +@RunWith(AndroidJUnit4.class) +public class EditorFileEncodingTest { + + private static final byte[] UTF_8_BOM = {(byte) 0xEF, (byte) 0xBB, (byte) 0xBF}; + private static final byte[] UTF_16LE_BOM = {(byte) 0xFF, (byte) 0xFE}; + private static final byte[] UTF_16BE_BOM = {(byte) 0xFE, (byte) 0xFF}; + private static final byte[] UTF_32LE_BOM = {(byte) 0xFF, (byte) 0xFE, 0, 0}; + + /** Cyrillic "Hello world" written as unicode escapes to stay encoding independent. */ + private static final String SAMPLE = "\u041f\u0440\u0438\u0432\u0435\u0442 \u043c\u0438\u0440"; + + private Context context; + + @Rule + public GrantPermissionRule mRuntimePermissionRule = GrantPermissionRule.grant( + Manifest.permission.WRITE_EXTERNAL_STORAGE, + Manifest.permission.READ_EXTERNAL_STORAGE); + + @Before + public void setUp() { + context = ApplicationProvider.getApplicationContext(); + clearRecovery(); + PreferenceManager.getDefaultSharedPreferences(context) + .edit() + .putString(SettingsService.SETTING_FILE_ENCODING, "UTF-8") + .putBoolean(SettingsService.SETTING_OPEN_LAST_FILE, false) + .commit(); + ServiceLocator.getInstance().getSettingsService(context).reloadSettings(context); + } + + @After + public void tearDown() { + clearRecovery(); + } + + @Test + public void utf8BomFileOpensWithoutBomCharAndSavesWithBom() throws Exception { + assertOpenEditSaveRoundTrip(SAMPLE, "UTF-8", UTF_8_BOM); + } + + @Test + public void utf16LeBomFileOpensAndRoundTrips() throws Exception { + assertOpenEditSaveRoundTrip(SAMPLE, "UTF-16LE", UTF_16LE_BOM); + } + + @Test + public void utf16BeBomFileOpensAndRoundTrips() throws Exception { + assertOpenEditSaveRoundTrip(SAMPLE, "UTF-16BE", UTF_16BE_BOM); + } + + @Test + public void utf32LeBomFileOpensAndRoundTrips() throws Exception { + Assume.assumeTrue(Charset.isSupported("UTF-32LE")); + assertOpenEditSaveRoundTrip(SAMPLE, "UTF-32LE", UTF_32LE_BOM); + } + + @Test + public void fileWithoutBomUsesConfiguredDefaultEncoding() throws Exception { + assertOpenEditSaveRoundTrip(SAMPLE, "UTF-8", new byte[0]); + } + + /** + * Creates a raw document, opens it in the editor, checks the decoded text + * (no stray BOM character), appends a character and saves, then checks the + * bytes on disk kept their encoding and byte order mark. + */ + private void assertOpenEditSaveRoundTrip(String content, String charsetName, byte[] expectedBom) + throws Exception { + Assume.assumeTrue(Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q); + Charset charset = Charset.forName(charsetName); + Uri documentUri = createTestDocumentRaw(concat(expectedBom, content.getBytes(charset))); + try { + Intent intent = new Intent(context, EditorActivity.class) + .setAction(Intent.ACTION_VIEW) + .setData(documentUri); + + try (ActivityScenario scenario = ActivityScenario.launch(intent)) { + onView(withId(R.id.editText1)).check(matches(withText(content))); + + scenario.onActivity(activity -> { + EditText editor = activity.findViewById(R.id.editText1); + editor.setSelection(editor.getText().length()); + editor.getText().append("!"); + }); + scenario.onActivity(activity -> invokeNoArgument(activity, "saveNamedFile")); + } + + byte[] savedBytes = readDocumentRawBytes(documentUri); + assertArrayEquals(concat(expectedBom, (content + "!").getBytes(charset)), savedBytes); + } finally { + context.getContentResolver().delete(documentUri, null, null); + } + } + + private static byte[] concat(byte[] prefix, byte[] suffix) { + byte[] result = new byte[prefix.length + suffix.length]; + System.arraycopy(prefix, 0, result, 0, prefix.length); + System.arraycopy(suffix, 0, result, prefix.length, suffix.length); + return result; + } + + private Uri createTestDocumentRaw(byte[] rawContent) throws Exception { + ContentValues values = new ContentValues(); + values.put(MediaStore.MediaColumns.DISPLAY_NAME, + "textpad-encoding-" + java.lang.System.nanoTime() + ".txt"); + values.put(MediaStore.MediaColumns.MIME_TYPE, "text/plain"); + values.put(MediaStore.MediaColumns.RELATIVE_PATH, "Download/TextPadTests"); + Uri uri = context.getContentResolver().insert(MediaStore.Downloads.EXTERNAL_CONTENT_URI, values); + if (uri == null) { + throw new IllegalStateException("Unable to create test document"); + } + try (java.io.OutputStream output = context.getContentResolver().openOutputStream(uri, "wt")) { + if (output == null) { + throw new IllegalStateException("Unable to write test document"); + } + output.write(rawContent); + } + return uri; + } + + private byte[] readDocumentRawBytes(Uri uri) throws Exception { + try (InputStream input = context.getContentResolver().openInputStream(uri); + ByteArrayOutputStream output = new ByteArrayOutputStream()) { + if (input == null) { + throw new IllegalStateException("Unable to read test document"); + } + byte[] buffer = new byte[1024]; + int count; + while ((count = input.read(buffer)) != -1) { + output.write(buffer, 0, count); + } + return output.toByteArray(); + } + } + + private static void invokeNoArgument(EditorActivity activity, String methodName) { + try { + Method method = EditorActivity.class.getDeclaredMethod(methodName); + method.setAccessible(true); + method.invoke(activity); + } catch (Exception error) { + throw new AssertionError(error); + } + } + + private void clearRecovery() { + context.getSharedPreferences("editor_recovery", Context.MODE_PRIVATE).edit().clear().commit(); + deleteRecursively(new RecoveryRepository(context).getDirectoryForTests()); + } + + private static void deleteRecursively(File file) { + if (!file.exists()) { + return; + } + File[] children = file.listFiles(); + if (children != null) { + for (File child : children) { + deleteRecursively(child); + } + } + file.delete(); + } +} diff --git a/app/src/androidTest/java/com/maxistar/textpad/test/EditorRecoveryTest.java b/app/src/androidTest/java/com/maxistar/textpad/test/EditorRecoveryTest.java index e081294..6552ac6 100644 --- a/app/src/androidTest/java/com/maxistar/textpad/test/EditorRecoveryTest.java +++ b/app/src/androidTest/java/com/maxistar/textpad/test/EditorRecoveryTest.java @@ -10,6 +10,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import android.Manifest; import android.content.Context; import android.content.ContentValues; import android.content.Intent; @@ -25,6 +26,7 @@ import androidx.test.core.app.ActivityScenario; import androidx.test.core.app.ApplicationProvider; import androidx.test.ext.junit.runners.AndroidJUnit4; +import androidx.test.rule.GrantPermissionRule; import com.maxistar.textpad.R; import com.maxistar.textpad.ServiceLocator; @@ -39,6 +41,7 @@ import org.junit.Before; import org.junit.Test; import org.junit.Assume; +import org.junit.Rule; import org.junit.runner.RunWith; import java.io.File; @@ -51,6 +54,11 @@ public class EditorRecoveryTest { private Context context; + @Rule + public GrantPermissionRule mRuntimePermissionRule = GrantPermissionRule.grant( + Manifest.permission.WRITE_EXTERNAL_STORAGE, + Manifest.permission.READ_EXTERNAL_STORAGE); + @Before public void setUp() { context = ApplicationProvider.getApplicationContext(); @@ -215,6 +223,50 @@ public void successfulSaveAsRemovesUntitledRecoveryAndUsesNamedIdentity() throws } } + @Test + public void utf16leWithBomRoundTripsThroughRecoveryRestoreAndSave() throws Exception { + Assume.assumeTrue(Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q); + String content = "Hello UTF-16LE"; + byte[] utf16leBytes = content.getBytes(java.nio.charset.Charset.forName("UTF-16LE")); + byte[] withBom = new byte[2 + utf16leBytes.length]; + withBom[0] = (byte) 0xFF; + withBom[1] = (byte) 0xFE; + java.lang.System.arraycopy(utf16leBytes, 0, withBom, 2, utf16leBytes.length); + + Uri documentUri = createTestDocumentRaw(withBom); + String key = RecoveryKeys.forDocumentUri(documentUri.toString()); + new RecoveryRepository(context).write( + metadataWithEncoding(key, documentUri.toString(), "UTF-16LE", true), + content + ); + + Intent intent = new Intent(context, EditorActivity.class) + .setAction(Intent.ACTION_VIEW) + .setData(documentUri); + + try (ActivityScenario scenario = ActivityScenario.launch(intent)) { + onView(withText(R.string.Restore)).perform(click()); + onView(withId(R.id.editText1)).check(matches(withText(content))); + scenario.onActivity(activity -> { + EditText editor = activity.findViewById(R.id.editText1); + editor.setSelection(content.length()); + editor.getText().append(" modified"); + }); + android.os.SystemClock.sleep(800); + scenario.recreate(); + onView(withText(R.string.Restore)).perform(click()); + scenario.onActivity(activity -> invokeNoArgument(activity, "saveNamedFile")); + byte[] savedBytes = readDocumentRawBytes(documentUri); + assertEquals(0xFF, savedBytes[0] & 0xFF); + assertEquals(0xFE, savedBytes[1] & 0xFF); + String savedText = new String(savedBytes, 2, savedBytes.length - 2, + java.nio.charset.Charset.forName("UTF-16LE")); + assertEquals("Hello UTF-16LE modified", savedText); + } finally { + context.getContentResolver().delete(documentUri, null, null); + } + } + private void assertLargeDocumentStateIsBinderSafe(boolean simpleScrolling) { setSimpleScrolling(simpleScrolling); String content = generatedDocument(1_050_000); @@ -261,6 +313,15 @@ private RecoveryMetadata metadata(String key, String documentUri) { ); } + private RecoveryMetadata metadataWithEncoding(String key, String documentUri, + String encoding, boolean hasBom) { + return new RecoveryMetadata( + key, documentUri, documentUri == null ? "newfile.txt" : "notes.txt", + documentUri == null, encoding, hasBom, + null, null, null, 0, 0, 0, 0, 1 + ); + } + private Uri createTestDocument(String content) throws Exception { ContentValues values = new ContentValues(); values.put(MediaStore.MediaColumns.DISPLAY_NAME, "textpad-recovery-" + java.lang.System.nanoTime() + ".txt"); @@ -296,6 +357,39 @@ private String readDocument(Uri uri) { } } + private Uri createTestDocumentRaw(byte[] rawContent) throws Exception { + ContentValues values = new ContentValues(); + values.put(MediaStore.MediaColumns.DISPLAY_NAME, "textpad-recovery-" + java.lang.System.nanoTime() + ".txt"); + values.put(MediaStore.MediaColumns.MIME_TYPE, "text/plain"); + values.put(MediaStore.MediaColumns.RELATIVE_PATH, "Download/TextPadTests"); + Uri uri = context.getContentResolver().insert(MediaStore.Downloads.EXTERNAL_CONTENT_URI, values); + if (uri == null) { + throw new IllegalStateException("Unable to create test document"); + } + try (java.io.OutputStream output = context.getContentResolver().openOutputStream(uri, "wt")) { + if (output == null) { + throw new IllegalStateException("Unable to write test document"); + } + output.write(rawContent); + } + return uri; + } + + private byte[] readDocumentRawBytes(Uri uri) throws Exception { + try (InputStream input = context.getContentResolver().openInputStream(uri); + ByteArrayOutputStream output = new ByteArrayOutputStream()) { + if (input == null) { + throw new IllegalStateException("Unable to read test document"); + } + byte[] buffer = new byte[1024]; + int count; + while ((count = input.read(buffer)) != -1) { + output.write(buffer, 0, count); + } + return output.toByteArray(); + } + } + private static void invokeNoArgument(EditorActivity activity, String methodName) { try { Method method = EditorActivity.class.getDeclaredMethod(methodName); diff --git a/app/src/androidTest/java/com/maxistar/textpad/test/recovery/RecoveryRepositoryTest.java b/app/src/androidTest/java/com/maxistar/textpad/test/recovery/RecoveryRepositoryTest.java index 1da645c..7a641f2 100644 --- a/app/src/androidTest/java/com/maxistar/textpad/test/recovery/RecoveryRepositoryTest.java +++ b/app/src/androidTest/java/com/maxistar/textpad/test/recovery/RecoveryRepositoryTest.java @@ -59,6 +59,31 @@ public void metadataRoundTripIgnoresUnknownFields() throws Exception { assertEquals(source.generation, restored.generation); } + @Test + public void metadataRoundTripsEncodingAndBom() throws Exception { + RecoveryMetadata source = new RecoveryMetadata( + RecoveryKeys.forDocumentUri("content://encoding-test/document/report.txt"), + "content://encoding-test/document/report.txt", + "report.txt", + false, + "UTF-16LE", + true, + 12L, + null, + "fingerprint", + 0, + 0, + 2, + 4, + 7 + ).published(5, "0123456789abcdef", 123456L); + + RecoveryMetadata restored = RecoveryMetadata.fromJson(source.toJson()); + + assertEquals("UTF-16LE", restored.encoding); + assertTrue(restored.hasBom); + } + @Test(expected = Exception.class) public void futureMetadataVersionIsRejected() throws Exception { JSONObject json = metadata(RecoveryKeys.forUntitledDocument(), null, 1).toJson(); diff --git a/app/src/main/java/com/maxistar/textpad/activities/EditorActivity.java b/app/src/main/java/com/maxistar/textpad/activities/EditorActivity.java index dc2795b..cec31f4 100644 --- a/app/src/main/java/com/maxistar/textpad/activities/EditorActivity.java +++ b/app/src/main/java/com/maxistar/textpad/activities/EditorActivity.java @@ -70,6 +70,7 @@ import com.maxistar.textpad.recovery.RecoveryWriter; import com.maxistar.textpad.utils.EditTextUndoRedo; import com.maxistar.textpad.utils.DocumentSaveValidator; +import com.maxistar.textpad.utils.FileEncoding; import com.maxistar.textpad.utils.FileNameHelper; import com.maxistar.textpad.utils.System; import com.maxistar.textpad.utils.TextConverter; @@ -130,6 +131,8 @@ public class EditorActivity extends AppCompatActivity { private ScrollView scrollView; private LinearLayout linearLayout; + private FileEncoding documentEncoding; + String urlFilename = TPStrings.EMPTY; Uri lastTriedSystemUri = null; @@ -261,6 +264,13 @@ private boolean simpleScrolling() { return settingsService.isUseSimpleScrolling(); } + private String resolveFileEncodingName() { + if (documentEncoding != null) { + return documentEncoding.getCharsetName(); + } + return settingsService.getFileEncoding(); + } + @RequiresApi(Build.VERSION_CODES.VANILLA_ICE_CREAM) private void applyEdgeToEdgeInsets() { View editorRoot = findViewById(R.id.editor_root); @@ -534,6 +544,9 @@ private void restoreDraft(RecoveryDraft draft) { originalSize = draft.metadata.originalSize; originalLastModified = draft.metadata.originalLastModified; originalContentSha256 = draft.metadata.originalContentSha256; + if (draft.metadata.encoding != null && !draft.metadata.encoding.isEmpty()) { + documentEncoding = FileEncoding.fromCharset(draft.metadata.encoding, draft.metadata.hasBom); + } setEditorText(draft.text, true); updateTitle(); if (!draft.metadata.untitled) { @@ -594,8 +607,8 @@ private RecoveryWriter.Snapshot createRecoverySnapshot() { identity, currentDisplayName(), identity == null, - settingsService.getFileEncoding(), - false, + resolveFileEncodingName(), + documentEncoding != null && documentEncoding.hasBom(), originalSize, originalLastModified, originalContentSha256, @@ -1159,6 +1172,7 @@ public void clearFile() { originalSize = null; originalLastModified = null; originalContentSha256 = null; + documentEncoding = null; selectionStart = 0; selectionEnd = 0; setEditorText(TPStrings.EMPTY, false); @@ -1340,7 +1354,7 @@ protected void saveFile(Uri uri) throws IOException { s = applyEndings(s); - outputStream.write(s.getBytes(settingsService.getFileEncoding())); + outputStream.write(FileEncoding.encode(s, documentEncoding, settingsService.getFileEncoding())); } finally { outputStream.close(); } @@ -1356,7 +1370,7 @@ private void guardedSaveNamedFile(boolean autosave) { SaveRequest request = new SaveRequest( editorGeneration, recoveryKey, - persistedText.getBytes(settingsService.getFileEncoding()) + FileEncoding.encode(persistedText, documentEncoding, settingsService.getFileEncoding()) ); boolean creatingDocument = nextSaveCreatesDocument || originalContentSha256 == null; nextSaveCreatesDocument = false; @@ -1551,8 +1565,11 @@ private void validateOpenDocumentOnForeground() { return; } - byte[] intendedBytes = applyEndings(mText.getText().toString()) - .getBytes(settingsService.getFileEncoding()); + byte[] intendedBytes = FileEncoding.encode( + applyEndings(mText.getText().toString()), + documentEncoding, + settingsService.getFileEncoding() + ); SaveRequest request = new SaveRequest(editorGeneration, recoveryKey, intendedBytes); DocumentSaveValidator.Outcome outcome = DocumentSaveValidator.classify( currentBytes, @@ -1572,7 +1589,8 @@ private void validateOpenDocumentOnForeground() { } private void applyExternalDocument(byte[] externalBytes) throws Exception { - String externalText = new String(externalBytes, settingsService.getFileEncoding()); + documentEncoding = FileEncoding.detect(externalBytes); + String externalText = FileEncoding.decode(externalBytes, documentEncoding, settingsService.getFileEncoding()); externalText = toUnixEndings(externalText); setEditorText(externalText, false); initEditor(); @@ -1588,8 +1606,11 @@ private void validateRestoredDraft() { return; } try { - byte[] intendedBytes = applyEndings(mText.getText().toString()) - .getBytes(settingsService.getFileEncoding()); + byte[] intendedBytes = FileEncoding.encode( + applyEndings(mText.getText().toString()), + documentEncoding, + settingsService.getFileEncoding() + ); SaveRequest request = new SaveRequest(editorGeneration, recoveryKey, intendedBytes); byte[] currentBytes = readNamedDocumentBytes(); DocumentSaveValidator.Outcome outcome = DocumentSaveValidator.classify( @@ -1664,9 +1685,8 @@ private void openNamedFileLegacyDirect(String filename) { dis.close(); fis.close(); - String ttt = new String(b, 0, length, - settingsService.getFileEncoding()); - + documentEncoding = FileEncoding.detect(b); + String ttt = FileEncoding.decode(b, documentEncoding, settingsService.getFileEncoding()); ttt = toUnixEndings(ttt); setEditorText(ttt, false); @@ -1717,7 +1737,8 @@ private void openNamedFileDirect(final Uri uri) { } byte[] b = bytes.toByteArray(); - String ttt = new String(b, settingsService.getFileEncoding()); + documentEncoding = FileEncoding.detect(b); + String ttt = FileEncoding.decode(b, documentEncoding, settingsService.getFileEncoding()); ttt = toUnixEndings(ttt); inputStream.close(); diff --git a/app/src/main/java/com/maxistar/textpad/recovery/RecoveryMetadata.java b/app/src/main/java/com/maxistar/textpad/recovery/RecoveryMetadata.java index 7772b46..1b11fcc 100644 --- a/app/src/main/java/com/maxistar/textpad/recovery/RecoveryMetadata.java +++ b/app/src/main/java/com/maxistar/textpad/recovery/RecoveryMetadata.java @@ -144,7 +144,7 @@ public JSONObject toJson() throws JSONException { value.put("documentUri", documentUri == null ? JSONObject.NULL : documentUri); value.put("displayName", displayName); value.put("isUntitled", untitled); - value.put("encoding", encoding); + value.put("encoding", encoding == null ? "" : encoding); value.put("hasBom", hasBom); value.put("originalSize", originalSize == null ? JSONObject.NULL : originalSize); value.put("originalLastModified", originalLastModified == null ? JSONObject.NULL : originalLastModified); @@ -169,7 +169,7 @@ public static RecoveryMetadata fromJson(JSONObject value) throws JSONException { nullableString(value, "documentUri"), value.optString("displayName", ""), value.getBoolean("isUntitled"), - value.optString("encoding", "UTF-8"), + nullableStringWithFallback(value, "encoding", "UTF-8"), value.optBoolean("hasBom", false), nullableLong(value, "originalSize"), nullableLong(value, "originalLastModified"), @@ -187,6 +187,14 @@ private static String nullableString(JSONObject value, String name) throws JSONE return value.isNull(name) ? null : value.getString(name); } + private static String nullableStringWithFallback(JSONObject value, String name, String fallback) throws JSONException { + if (value.isNull(name) || !value.has(name)) { + return fallback; + } + String result = value.getString(name); + return result == null || result.isEmpty() ? fallback : result; + } + private static Long nullableLong(JSONObject value, String name) throws JSONException { return value.isNull(name) ? null : value.getLong(name); } diff --git a/app/src/main/java/com/maxistar/textpad/utils/FileEncoding.java b/app/src/main/java/com/maxistar/textpad/utils/FileEncoding.java new file mode 100644 index 0000000..cdbfc2a --- /dev/null +++ b/app/src/main/java/com/maxistar/textpad/utils/FileEncoding.java @@ -0,0 +1,135 @@ +package com.maxistar.textpad.utils; + +import java.nio.charset.Charset; + +/** + * Detects a text file encoding from its byte order mark (BOM) and decodes or + * encodes the content keeping the original encoding. + */ +public class FileEncoding { + + public static final String UTF_8 = "UTF-8"; + public static final String UTF_16LE = "UTF-16LE"; + public static final String UTF_16BE = "UTF-16BE"; + public static final String UTF_32LE = "UTF-32LE"; + public static final String UTF_32BE = "UTF-32BE"; + + private static final byte[] BOM_UTF_32LE = {(byte) 0xFF, (byte) 0xFE, 0, 0}; + private static final byte[] BOM_UTF_32BE = {0, 0, (byte) 0xFE, (byte) 0xFF}; + private static final byte[] BOM_UTF_8 = {(byte) 0xEF, (byte) 0xBB, (byte) 0xBF}; + private static final byte[] BOM_UTF_16BE = {(byte) 0xFE, (byte) 0xFF}; + private static final byte[] BOM_UTF_16LE = {(byte) 0xFF, (byte) 0xFE}; + + private final String charsetName; + private final byte[] bom; + + private FileEncoding(String charsetName, byte[] bom) { + this.charsetName = charsetName; + this.bom = bom; + } + + public String getCharsetName() { + return charsetName; + } + + public byte[] getBom() { + return bom; + } + + public boolean hasBom() { + return bom != null; + } + + public static FileEncoding detect(byte[] bytes) { + if (bytes == null || bytes.length == 0) { + return null; + } + if (startsWith(bytes, BOM_UTF_32LE)) { + return new FileEncoding(UTF_32LE, BOM_UTF_32LE); + } + if (startsWith(bytes, BOM_UTF_32BE)) { + return new FileEncoding(UTF_32BE, BOM_UTF_32BE); + } + if (startsWith(bytes, BOM_UTF_8)) { + return new FileEncoding(UTF_8, BOM_UTF_8); + } + if (startsWith(bytes, BOM_UTF_16BE)) { + return new FileEncoding(UTF_16BE, BOM_UTF_16BE); + } + if (startsWith(bytes, BOM_UTF_16LE)) { + return new FileEncoding(UTF_16LE, BOM_UTF_16LE); + } + return null; + } + + public static FileEncoding fromCharset(String charsetName, boolean hasBom) { + if (charsetName == null || charsetName.isEmpty()) { + return null; + } + if (!hasBom) { + return new FileEncoding(charsetName, null); + } + byte[] bom = bomForCharset(charsetName); + return new FileEncoding(charsetName, bom); + } + + private static byte[] bomForCharset(String charsetName) { + if (UTF_32LE.equals(charsetName)) return BOM_UTF_32LE; + if (UTF_32BE.equals(charsetName)) return BOM_UTF_32BE; + if (UTF_8.equals(charsetName)) return BOM_UTF_8; + if (UTF_16BE.equals(charsetName)) return BOM_UTF_16BE; + if (UTF_16LE.equals(charsetName)) return BOM_UTF_16LE; + return null; + } + + public static String decode(byte[] bytes, FileEncoding encoding, String fallbackCharsetName) { + if (bytes == null) { + return ""; + } + int offset = 0; + if (encoding != null && encoding.hasBom() && bytes.length >= encoding.getBom().length) { + offset = encoding.getBom().length; + } + String charsetName = encoding != null ? encoding.getCharsetName() : fallbackCharsetName; + try { + return new String(bytes, offset, bytes.length - offset, charsetForName(charsetName)); + } catch (Exception e) { + return new String(bytes); + } + } + + public static byte[] encode(String text, FileEncoding encoding, String fallbackCharsetName) { + String charsetName = encoding != null ? encoding.getCharsetName() : fallbackCharsetName; + byte[] body = text.getBytes(charsetForName(charsetName)); + if (encoding != null && encoding.hasBom()) { + byte[] result = new byte[encoding.getBom().length + body.length]; + java.lang.System.arraycopy(encoding.getBom(), 0, result, 0, encoding.getBom().length); + java.lang.System.arraycopy(body, 0, result, encoding.getBom().length, body.length); + return result; + } + return body; + } + + private static Charset charsetForName(String charsetName) { + if (charsetName == null) { + return Charset.defaultCharset(); + } + try { + return Charset.forName(charsetName); + } catch (Exception e) { + return Charset.defaultCharset(); + } + } + + private static boolean startsWith(byte[] bytes, byte[] prefix) { + if (bytes.length < prefix.length) { + return false; + } + for (int i = 0; i < prefix.length; i++) { + if (bytes[i] != prefix[i]) { + return false; + } + } + return true; + } +} \ No newline at end of file diff --git a/app/src/test/java/com/maxistar/textpad/utils/FileEncodingTest.java b/app/src/test/java/com/maxistar/textpad/utils/FileEncodingTest.java new file mode 100644 index 0000000..c0ba5bf --- /dev/null +++ b/app/src/test/java/com/maxistar/textpad/utils/FileEncodingTest.java @@ -0,0 +1,192 @@ +package com.maxistar.textpad.utils; + +import java.io.ByteArrayOutputStream; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class FileEncodingTest { + + private static byte[] bom(byte[]... parts) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + for (byte[] part : parts) { + out.write(part, 0, part.length); + } + return out.toByteArray(); + } + + @Test + public void detectUtf16LeBom() { + FileEncoding encoding = FileEncoding.detect(new byte[]{(byte) 0xFF, (byte) 0xFE, 0x41, 0x00}); + assertEquals(FileEncoding.UTF_16LE, encoding.getCharsetName()); + assertEquals(2, encoding.getBom().length); + } + + @Test + public void detectUtf16BeBom() { + FileEncoding encoding = FileEncoding.detect(new byte[]{(byte) 0xFE, (byte) 0xFF, 0x00, 0x41}); + assertEquals(FileEncoding.UTF_16BE, encoding.getCharsetName()); + } + + @Test + public void detectUtf8Bom() { + FileEncoding encoding = FileEncoding.detect(new byte[]{(byte) 0xEF, (byte) 0xBB, (byte) 0xBF, 0x41}); + assertEquals(FileEncoding.UTF_8, encoding.getCharsetName()); + } + + @Test + public void detectUtf32LeBom() { + FileEncoding encoding = FileEncoding.detect(new byte[]{(byte) 0xFF, (byte) 0xFE, 0, 0, 0x41, 0, 0, 0}); + assertEquals(FileEncoding.UTF_32LE, encoding.getCharsetName()); + } + + @Test + public void detectUtf32BeBom() { + FileEncoding encoding = FileEncoding.detect(new byte[]{0, 0, (byte) 0xFE, (byte) 0xFF, 0, 0, 0, 0x41}); + assertEquals(FileEncoding.UTF_32BE, encoding.getCharsetName()); + } + + @Test + public void detectNoBom() { + assertNull(FileEncoding.detect("hello".getBytes(StandardCharsets.UTF_8))); + assertNull(FileEncoding.detect(new byte[0])); + assertNull(FileEncoding.detect(null)); + } + + @Test + public void decodeStripsBom() { + byte[] bytes = bom(new byte[]{(byte) 0xFF, (byte) 0xFE}, + "\u041f\u0440\u0438\u0432\u0435\u0442".getBytes(StandardCharsets.UTF_16LE)); + FileEncoding encoding = FileEncoding.detect(bytes); + String text = FileEncoding.decode(bytes, encoding, FileEncoding.UTF_8); + assertEquals("\u041f\u0440\u0438\u0432\u0435\u0442", text); + } + + @Test + public void decodeWithoutBomUsesFallback() { + byte[] bytes = "hello".getBytes(StandardCharsets.UTF_8); + String text = FileEncoding.decode(bytes, null, FileEncoding.UTF_8); + assertEquals("hello", text); + } + + @Test + public void encodeRestoresBom() { + byte[] bytes = bom(new byte[]{(byte) 0xFF, (byte) 0xFE}, + "\u041f\u0440\u0438\u0432\u0435\u0442".getBytes(StandardCharsets.UTF_16LE)); + FileEncoding encoding = FileEncoding.detect(bytes); + byte[] encoded = FileEncoding.encode("\u041f\u0440\u0438\u0432\u0435\u0442", encoding, FileEncoding.UTF_8); + assertArrayEquals(bytes, encoded); + } + + @Test + public void encodeRoundTrip() { + byte[] bytes = bom(new byte[]{(byte) 0xEF, (byte) 0xBB, (byte) 0xBF}, + "test".getBytes(StandardCharsets.UTF_8)); + FileEncoding encoding = FileEncoding.detect(bytes); + String text = FileEncoding.decode(bytes, encoding, FileEncoding.UTF_8); + byte[] encoded = FileEncoding.encode(text, encoding, FileEncoding.UTF_8); + assertArrayEquals(bytes, encoded); + } + + @Test + public void fromCharsetReconstructsBomForEachSupportedCharset() { + assertArrayEquals(new byte[]{(byte) 0xEF, (byte) 0xBB, (byte) 0xBF}, + FileEncoding.fromCharset(FileEncoding.UTF_8, true).getBom()); + assertArrayEquals(new byte[]{(byte) 0xFF, (byte) 0xFE}, + FileEncoding.fromCharset(FileEncoding.UTF_16LE, true).getBom()); + assertArrayEquals(new byte[]{(byte) 0xFE, (byte) 0xFF}, + FileEncoding.fromCharset(FileEncoding.UTF_16BE, true).getBom()); + assertArrayEquals(new byte[]{(byte) 0xFF, (byte) 0xFE, 0, 0}, + FileEncoding.fromCharset(FileEncoding.UTF_32LE, true).getBom()); + assertArrayEquals(new byte[]{0, 0, (byte) 0xFE, (byte) 0xFF}, + FileEncoding.fromCharset(FileEncoding.UTF_32BE, true).getBom()); + assertTrue(FileEncoding.fromCharset(FileEncoding.UTF_16LE, true).hasBom()); + } + + @Test + public void fromCharsetWithoutBomReturnsNullBom() { + FileEncoding encoding = FileEncoding.fromCharset(FileEncoding.UTF_16LE, false); + assertEquals(FileEncoding.UTF_16LE, encoding.getCharsetName()); + assertNull(encoding.getBom()); + assertFalse(encoding.hasBom()); + } + + @Test + public void fromCharsetRejectsNullOrEmptyName() { + assertNull(FileEncoding.fromCharset(null, true)); + assertNull(FileEncoding.fromCharset("", false)); + } + + @Test + public void fromCharsetUnknownCharsetWithBomHasNoBom() { + FileEncoding encoding = FileEncoding.fromCharset("NO_SUCH_CHARSET", true); + assertEquals("NO_SUCH_CHARSET", encoding.getCharsetName()); + assertNull(encoding.getBom()); + assertFalse(encoding.hasBom()); + } + + @Test + public void detectIgnoresTruncatedBom() { + assertNull(FileEncoding.detect(new byte[]{(byte) 0xFF})); + assertNull(FileEncoding.detect(new byte[]{(byte) 0xEF, (byte) 0xBB})); + assertNull(FileEncoding.detect(new byte[]{(byte) 0xFE})); + assertNull(FileEncoding.detect(new byte[]{0, 0, (byte) 0xFE})); + assertEquals(FileEncoding.UTF_16LE, + FileEncoding.detect(new byte[]{(byte) 0xFF, (byte) 0xFE, 0}).getCharsetName()); + } + + @Test + public void decodeBomOnlyFileReturnsEmpty() { + byte[] utf16LeBomOnly = new byte[]{(byte) 0xFF, (byte) 0xFE}; + FileEncoding encoding = FileEncoding.detect(utf16LeBomOnly); + assertEquals(FileEncoding.UTF_16LE, encoding.getCharsetName()); + assertEquals("", FileEncoding.decode(utf16LeBomOnly, encoding, FileEncoding.UTF_8)); + + byte[] utf8BomOnly = new byte[]{(byte) 0xEF, (byte) 0xBB, (byte) 0xBF}; + assertEquals("", FileEncoding.decode(utf8BomOnly, FileEncoding.detect(utf8BomOnly), + FileEncoding.UTF_8)); + } + + @Test + public void encodeWithNullEncodingUsesFallback() { + byte[] encoded = FileEncoding.encode("hello", null, FileEncoding.UTF_8); + assertArrayEquals("hello".getBytes(StandardCharsets.UTF_8), encoded); + } + + @Test + public void encodeWithoutBomDoesNotPrefixSignature() { + FileEncoding encoding = FileEncoding.fromCharset(FileEncoding.UTF_16LE, false); + byte[] encoded = FileEncoding.encode("hi", encoding, FileEncoding.UTF_8); + assertArrayEquals("hi".getBytes(Charset.forName(FileEncoding.UTF_16LE)), encoded); + } + + @Test + public void encodeRoundTripForAllSupportedEncodings() { + String text = "\u041f\u0440\u0438\u0432\u0435\u0442 \u043c\u0438\u0440"; + assertRoundTrip(FileEncoding.UTF_8, text); + assertRoundTrip(FileEncoding.UTF_16LE, text); + assertRoundTrip(FileEncoding.UTF_16BE, text); + assertRoundTrip(FileEncoding.UTF_32LE, text); + assertRoundTrip(FileEncoding.UTF_32BE, text); + } + + private void assertRoundTrip(String charsetName, String text) { + FileEncoding encoding = FileEncoding.fromCharset(charsetName, true); + byte[] file = FileEncoding.encode(text, encoding, FileEncoding.UTF_8); + + FileEncoding detected = FileEncoding.detect(file); + assertEquals(charsetName, detected.getCharsetName()); + + String decoded = FileEncoding.decode(file, detected, FileEncoding.UTF_8); + assertEquals(text, decoded); + + assertArrayEquals(file, FileEncoding.encode(decoded, detected, FileEncoding.UTF_8)); + } +} \ No newline at end of file