Add overflow protections based on testing moshix' bbs

lib3270j/src/test/java/haus/nightmare/lib3270j/datastream/FastByteBufferTest.java

@@ -151,4 +151,39 @@ public class FastByteBufferTest {
         executor.shutdown();
         assertEquals(0, failures.get(), "No failures occurred during concurrent pool operations");
     }
+
+    @Test
+    @DisplayName("FastByteBuffer enforces maxCapacity and throws FramingBufferOverflowException")
+    public void testFastByteBufferMaxCapacityEnforcement() {
+        FastByteBuffer buf = new FastByteBuffer(64, 128);
+        assertEquals(128, buf.getMaxCapacity());
+        assertEquals(64, buf.getInitialCapacity());
+
+        byte[] chunk = new byte[64];
+        buf.write(chunk, 0, 64);
+        assertEquals(64, buf.size());
+
+        buf.write(chunk, 0, 64);
+        assertEquals(128, buf.size());
+
+        // Writing one more byte must throw FramingBufferOverflowException
+        assertThrows(FramingBufferOverflowException.class, () -> buf.write(0xFF));
+
+        // Writing a chunk that exceeds capacity must also throw
+        assertThrows(FramingBufferOverflowException.class, () -> buf.write(chunk, 0, 10));
+
+        // After reset, buffer is ready for use again
+        buf.reset();
+        assertEquals(0, buf.size());
+        buf.write(chunk, 0, 64);
+        assertEquals(64, buf.size());
+    }
+
+    @Test
+    @DisplayName("FastByteBuffer default maxCapacity is 1048576 bytes (1MB)")
+    public void testDefaultMaxCapacity() {
+        FastByteBuffer buf = new FastByteBuffer();
+        assertEquals(FastByteBuffer.DEFAULT_MAX_CAPACITY, buf.getMaxCapacity());
+        assertEquals(1048576, buf.getMaxCapacity());
+    }
 }

lib3270j/src/test/java/haus/nightmare/lib3270j/telnet/FramingBufferBoundedCapTest.java

@@ -0,0 +1,223 @@
+package haus.nightmare.lib3270j.telnet;
+
+import haus.nightmare.lib3270j.ConnectionConfig;
+import haus.nightmare.lib3270j.ConnectionState;
+import haus.nightmare.lib3270j.TerminalModel;
+import haus.nightmare.lib3270j.charset.EbcdicTranslator;
+import haus.nightmare.lib3270j.datastream.DataStreamProcessor;
+import haus.nightmare.lib3270j.nvt.NvtProcessor;
+import haus.nightmare.lib3270j.printer.PrinterConfig;
+import haus.nightmare.lib3270j.printer.Telnet3270EP;
+import haus.nightmare.lib3270j.protocol.TelnetConstants;
+import haus.nightmare.lib3270j.screen.ScreenBuffer;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import static haus.nightmare.lib3270j.protocol.TelnetConstants.*;
+import static org.junit.jupiter.api.Assertions.*;
+
+public class FramingBufferBoundedCapTest {
+
+    private ConnectionConfig config;
+    private ScreenBuffer screenBuffer;
+    private DataStreamProcessor dsProcessor;
+    private TelnetFSM fsm;
+    private MockConnection connection;
+
+    private static class MockConnection extends TelnetConnection {
+        final List<byte[]> sentData = new ArrayList<>();
+
+        MockConnection(ConnectionConfig config, TelnetFSM fsm) {
+            super(config, fsm);
+        }
+
+        @Override
+        public synchronized void sendRaw(byte[] data) {
+            sentData.add(data.clone());
+        }
+
+        @Override
+        public synchronized void sendRaw(byte[] data, int offset, int length) {
+            byte[] b = new byte[length];
+            System.arraycopy(data, offset, b, 0, length);
+            sentData.add(b);
+        }
+    }
+
+    @BeforeEach
+    public void setup() {
+        config = new ConnectionConfig("localhost", 23, TerminalModel.IBM_3279_4);
+        screenBuffer = new ScreenBuffer(TerminalModel.IBM_3279_4, new EbcdicTranslator());
+        dsProcessor = new DataStreamProcessor(screenBuffer, new EbcdicTranslator());
+        fsm = new TelnetFSM(config, screenBuffer, dsProcessor);
+        haus.nightmare.lib3270j.input.InputProcessor inputProcessor =
+                new haus.nightmare.lib3270j.input.InputProcessor(screenBuffer, new EbcdicTranslator(), fsm);
+        dsProcessor.setInputProcessor(inputProcessor);
+        connection = new MockConnection(config, fsm);
+        fsm.setConnection(connection);
+    }
+
+    private void feedBytes(int... bytes) {
+        byte[] b = new byte[bytes.length];
+        for (int i = 0; i < bytes.length; i++) {
+            b[i] = (byte) bytes[i];
+        }
+        fsm.feedBytes(b, 0, b.length);
+    }
+
+    @Test
+    @DisplayName("Framing buffer caps at 1MB and enters discard mode if stream lacks IAC EOR")
+    public void testFramingBufferOverflowDiscardsUntilEor() {
+        fsm.onConnected();
+
+        // Negotiate BINARY and EOR to establish standard 3270 streaming regime
+        feedBytes(IAC, DO, TELOPT_EOR);
+        feedBytes(IAC, WILL, TELOPT_EOR);
+        feedBytes(IAC, DO, TELOPT_BINARY);
+        feedBytes(IAC, WILL, TELOPT_BINARY);
+
+        assertEquals(1048576, fsm.getMaxFramingBufferSize(), "Default framing buffer cap should be 1MB (1048576 bytes)");
+        assertFalse(fsm.isRecordDiscardMode());
+
+        // Feed chunks totaling over 1MB without any IAC EOR delimiter
+        byte[] chunk = new byte[8192];
+        Arrays.fill(chunk, (byte) 0xC1); // EBCDIC 'A'
+
+        for (int i = 0; i < 128; i++) {
+            fsm.feedBytes(chunk, 0, chunk.length); // 128 * 8192 = 1,048,576 bytes
+        }
+        assertFalse(fsm.isRecordDiscardMode(), "Buffer should be at exact cap without overflowing yet");
+
+        // Now feed an additional chunk exceeding the 1MB cap without IAC EOR
+        fsm.feedBytes(chunk, 0, 100);
+
+        assertTrue(fsm.isRecordDiscardMode(), "FSM must enter recordDiscardMode when cap is exceeded without IAC EOR");
+        assertEquals(0, fsm.getCurrentFramingBufferSize(), "Buffer must be reset to free memory upon overflow");
+
+        // Further incoming data without IAC EOR must be discarded without growing the buffer
+        fsm.feedBytes(chunk, 0, chunk.length);
+        assertEquals(0, fsm.getCurrentFramingBufferSize(), "Discard mode must drop subsequent data without accumulating memory");
+        assertTrue(fsm.isRecordDiscardMode());
+
+        // Feeding IAC EOR must clear discard mode and reset buffer cleanly
+        feedBytes(IAC, EOR);
+        assertFalse(fsm.isRecordDiscardMode(), "IAC EOR must clear recordDiscardMode");
+        assertEquals(0, fsm.getCurrentFramingBufferSize());
+
+        // A subsequent valid record must be processed normally
+        // 0xF5 (Erase/Write), 0xC2 (WCC: unlock keyboard), 0x11 (SBA), 0x00, 0x00, 0xC1 ('A'), IAC, EOR
+        byte[] valid3270 = new byte[] { (byte) 0xF5, (byte) 0xC2, (byte) 0x11, 0x00, 0x00, (byte) 0xC1, (byte) IAC, (byte) EOR };
+        fsm.feedBytes(valid3270, 0, valid3270.length);
+
+        assertEquals('A', screenBuffer.getDisplayCell(0).ucs4);
+    }
+
+    @Test
+    @DisplayName("Custom maxFramingBufferSize on ConnectionConfig is respected by TelnetFSM")
+    public void testCustomMaxFramingBufferSize() {
+        ConnectionConfig customConfig = new ConnectionConfig("localhost", 23, TerminalModel.IBM_3279_4);
+        customConfig.setMaxFramingBufferSize(1024); // 1KB custom limit
+
+        TelnetFSM customFsm = new TelnetFSM(customConfig, screenBuffer, dsProcessor);
+        customFsm.setConnection(connection);
+        customFsm.onConnected();
+
+        assertEquals(1024, customFsm.getMaxFramingBufferSize());
+
+        // Negotiate BINARY and EOR
+        byte[] neg = new byte[] {
+            (byte) IAC, (byte) DO, (byte) TELOPT_EOR,
+            (byte) IAC, (byte) WILL, (byte) TELOPT_EOR,
+            (byte) IAC, (byte) DO, (byte) TELOPT_BINARY,
+            (byte) IAC, (byte) WILL, (byte) TELOPT_BINARY
+        };
+        customFsm.feedBytes(neg, 0, neg.length);
+
+        byte[] payload500 = new byte[500];
+        Arrays.fill(payload500, (byte) 0xC1);
+        customFsm.feedBytes(payload500, 0, 500);
+        assertEquals(500, customFsm.getCurrentFramingBufferSize());
+        assertFalse(customFsm.isRecordDiscardMode());
+
+        customFsm.feedBytes(payload500, 0, 500);
+        assertEquals(1000, customFsm.getCurrentFramingBufferSize());
+        assertFalse(customFsm.isRecordDiscardMode());
+
+        // Overflow over 1024 limit
+        customFsm.feedBytes(payload500, 0, 100);
+        assertTrue(customFsm.isRecordDiscardMode());
+        assertEquals(0, customFsm.getCurrentFramingBufferSize());
+    }
+
+    @Test
+    @DisplayName("Telnet subnegotiation buffer caps at 8KB if stream lacks IAC SE")
+    public void testSubnegotiationBufferCap() {
+        fsm.onConnected();
+
+        // Begin subnegotiation: IAC SB TELOPT_TTYPE
+        feedBytes(IAC, SB, TELOPT_TTYPE);
+
+        // Feed > 8192 bytes without IAC SE
+        byte[] largeSb = new byte[8200];
+        Arrays.fill(largeSb, (byte) 'X');
+        fsm.feedBytes(largeSb, 0, largeSb.length);
+
+        // Feed a regular byte: should be back in TNS_DATA, not accumulating in SB
+        feedBytes(IAC, NOP);
+        // Ensure no exception and session remains operational
+        assertTrue(true);
+    }
+
+    @Test
+    @DisplayName("Printer framing buffer caps at 1MB and enters discard mode without IAC EOR")
+    public void testPrinterFramingBufferCap() throws Exception {
+        PrinterConfig pcfg = new PrinterConfig("localhost", 23);
+        assertEquals(1048576, pcfg.getMaxFramingBufferSize());
+
+        Telnet3270EP printer = new Telnet3270EP(pcfg);
+        assertEquals(1048576, printer.getMaxFramingBufferSize());
+        assertFalse(printer.isRecordDiscardMode());
+
+        Method processIncoming = Telnet3270EP.class.getDeclaredMethod("processIncomingBytes", byte[].class, int.class, int.class);
+        processIncoming.setAccessible(true);
+
+        byte[] chunk = new byte[32768];
+        Arrays.fill(chunk, (byte) 0x40);
+
+        for (int i = 0; i < 33; i++) {
+            processIncoming.invoke(printer, chunk, 0, chunk.length); // 33 * 32768 = 1,081,344 bytes > 1,048,576
+        }
+
+        assertTrue(printer.isRecordDiscardMode(), "Printer must enter discard mode when record exceeds 1MB");
+
+        // Send IAC EOR
+        byte[] eor = new byte[] { (byte) IAC, (byte) EOR };
+        processIncoming.invoke(printer, eor, 0, eor.length);
+
+        assertFalse(printer.isRecordDiscardMode(), "Printer must clear discard mode on IAC EOR");
+    }
+
+    @Test
+    @DisplayName("NvtProcessor caps escape sequence buffer at 4KB if terminator is missing")
+    public void testNvtProcessorEscapeBufferCap() {
+        NvtProcessor nvt = fsm.getNvtProcessor();
+
+        // Stream ESC [ followed by 5000 digits without terminating byte (0x40..0x7E)
+        byte[] escStart = new byte[] { 0x1B, '[' };
+        nvt.processNVTData(escStart, 0, escStart.length);
+
+        byte[] digits = new byte[5000];
+        Arrays.fill(digits, (byte) '1');
+        nvt.processNVTData(digits, 0, digits.length);
+
+        // After escBuffer reaches 4096 bytes, it aborts escape parsing and returns to STATE_NORMAL.
+        // Subsequent digits are rendered as plain text in the screen buffer starting at cell 0.
+        assertEquals('1', screenBuffer.getCell(0).ucs4);
+    }
+}