From 1ceb380f6f2c9d9a52026416f3573111c2a53fba Mon Sep 17 00:00:00 2001 From: CTCaer Date: Sat, 4 Jul 2020 22:12:12 +0300 Subject: [PATCH] Add hekate loader --- .gitignore | 4 + Makefile | 29 ++- loader/Makefile | 62 +++++ loader/link.ld | 22 ++ loader/loader.c | 107 +++++++++ loader/start.S | 73 ++++++ tools/bin2c/Makefile | 12 + tools/bin2c/bin2c.c | 77 ++++++ tools/lz/Makefile | 12 + tools/lz/lz.c | 546 +++++++++++++++++++++++++++++++++++++++++++ tools/lz/lz.h | 61 +++++ tools/lz/lz77.c | 93 ++++++++ 12 files changed, 1095 insertions(+), 3 deletions(-) create mode 100644 loader/Makefile create mode 100644 loader/link.ld create mode 100644 loader/loader.c create mode 100644 loader/start.S create mode 100644 tools/bin2c/Makefile create mode 100644 tools/bin2c/bin2c.c create mode 100644 tools/lz/Makefile create mode 100644 tools/lz/lz.c create mode 100644 tools/lz/lz.h create mode 100644 tools/lz/lz77.c diff --git a/.gitignore b/.gitignore index 30e95f0..f634a9b 100755 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,7 @@ .vscode build/* output/* +loader/payload_00.h +loader/payload_01.h +tools/bin2c/bin2c +tools/lz/lz77 diff --git a/Makefile b/Makefile index 7a7621e..9543aad 100755 --- a/Makefile +++ b/Makefile @@ -78,14 +78,23 @@ LDFLAGS = $(ARCH) -nostartfiles -lgcc -Wl,--nmagic,--gc-sections -Xlinker --defs MODULEDIRS := $(wildcard modules/*) NYXDIR := $(wildcard nyx) +LDRDIR := $(wildcard loader) +TOOLSLZ := $(wildcard tools/lz) +TOOLSB2C := $(wildcard tools/bin2c) +TOOLS := $(TOOLSLZ) $(TOOLSB2C) ################################################################################ -.PHONY: all clean $(MODULEDIRS) $(NYXDIR) +.PHONY: all clean $(MODULEDIRS) $(NYXDIR) $(LDRDIR) $(TOOLS) -all: $(TARGET).bin +all: $(TARGET).bin $(LDRDIR) @printf ICTC49 >> $(OUTPUTDIR)/$(TARGET).bin @echo "--------------------------------------" + @echo -n "Uncompr size: " + $(eval BIN_SIZE = $(shell wc -c < $(OUTPUTDIR)/$(TARGET)_unc.bin)) + @echo $(BIN_SIZE)" Bytes" + @echo "Uncompr Max: 140288 Bytes + 3 KiB BSS" + @if [ ${BIN_SIZE} -gt 140288 ]; then echo "\e[1;33mUncompr size exceeds limit!\e[0m"; fi @echo -n "Payload size: " $(eval BIN_SIZE = $(shell wc -c < $(OUTPUTDIR)/$(TARGET).bin)) @echo $(BIN_SIZE)" Bytes" @@ -104,7 +113,21 @@ $(MODULEDIRS): $(NYXDIR): @$(MAKE) --no-print-directory -C $@ $(MAKECMDGOALS) -$(MAKEFLAGS) -$(TARGET).bin: $(BUILDDIR)/$(TARGET)/$(TARGET).elf $(MODULEDIRS) $(NYXDIR) +$(LDRDIR): $(TARGET).bin + @$(TOOLSLZ)/lz77 $(OUTPUTDIR)/$(TARGET).bin + mv $(OUTPUTDIR)/$(TARGET).bin $(OUTPUTDIR)/$(TARGET)_unc.bin + @mv $(OUTPUTDIR)/$(TARGET).bin.00.lz payload_00 + @mv $(OUTPUTDIR)/$(TARGET).bin.01.lz payload_01 + @$(TOOLSB2C)/bin2c payload_00 > $(LDRDIR)/payload_00.h + @$(TOOLSB2C)/bin2c payload_01 > $(LDRDIR)/payload_01.h + @rm payload_00 + @rm payload_01 + @$(MAKE) --no-print-directory -C $@ $(MAKECMDGOALS) -$(MAKEFLAGS) PAYLOAD_NAME=$(TARGET) + +$(TOOLS): + @$(MAKE) --no-print-directory -C $@ $(MAKECMDGOALS) -$(MAKEFLAGS) + +$(TARGET).bin: $(BUILDDIR)/$(TARGET)/$(TARGET).elf $(MODULEDIRS) $(NYXDIR) $(TOOLS) $(OBJCOPY) -S -O binary $< $(OUTPUTDIR)/$@ $(BUILDDIR)/$(TARGET)/$(TARGET).elf: $(OBJS) diff --git a/loader/Makefile b/loader/Makefile new file mode 100644 index 0000000..2fcc110 --- /dev/null +++ b/loader/Makefile @@ -0,0 +1,62 @@ +ifeq ($(strip $(DEVKITARM)),) +$(error "Please set DEVKITARM in your environment. export DEVKITARM=devkitARM") +endif + +include $(DEVKITARM)/base_rules + +################################################################################ + +LDR_LOAD_ADDR := 0x40007000 +IPL_MAGIC := 0x43544349 #"ICTC" +include ../Versions.inc + +################################################################################ + +TARGET := loader +BUILDDIR := ../build +OUTPUTDIR := ../output +BDKDIR := bdk +BDKINC := -I../$(BDKDIR) +VPATH += $(dir $(wildcard ../$(BDKDIR)/*/)) $(dir $(wildcard ../$(BDKDIR)/*/*/)) + +# Main and graphics. +OBJS = $(addprefix $(BUILDDIR)/$(TARGET)/, \ + start.o loader.o lz.o \ +) + +################################################################################ + +CUSTOMDEFINES := -DBL_MAGIC=$(IPL_MAGIC) +CUSTOMDEFINES += -DBL_VER_MJ=$(BLVERSION_MAJOR) -DBL_VER_MN=$(BLVERSION_MINOR) -DBL_VER_HF=$(BLVERSION_HOTFX) -DBL_RESERVED=$(BLVERSION_RSVD) + +ARCH := -march=armv4t -mtune=arm7tdmi -mthumb-interwork +CFLAGS = $(ARCH) -O2 -g -nostdlib -ffunction-sections -fdata-sections -fomit-frame-pointer -std=gnu11 -Wall $(CUSTOMDEFINES) +LDFLAGS = $(ARCH) -nostartfiles -lgcc -Wl,--nmagic,--gc-sections -Xlinker --defsym=LDR_LOAD_ADDR=$(LDR_LOAD_ADDR) + +################################################################################ + +.PHONY: all clean + +all: $(TARGET).bin $(TOOLSLZ) $(TOOLSB2C) + +clean: + @rm -rf $(OBJS) + +$(TARGET).bin: $(BUILDDIR)/$(TARGET)/$(TARGET).elf + $(OBJCOPY) -S -O binary $< $(OUTPUTDIR)/$(PAYLOAD_NAME).bin + +$(BUILDDIR)/$(TARGET)/$(TARGET).elf: $(OBJS) + @$(CC) $(LDFLAGS) -T link.ld $^ -o $@ + +$(BUILDDIR)/$(TARGET)/%.o: %.c + @$(CC) $(CFLAGS) $(BDKINC) -c $< -o $@ + +$(BUILDDIR)/$(TARGET)/%.o: %.S + @$(CC) $(CFLAGS) -c $< -o $@ + +$(OBJS): $(BUILDDIR)/$(TARGET) + +$(BUILDDIR)/$(TARGET): + @mkdir -p "$(BUILDDIR)" + @mkdir -p "$(BUILDDIR)/$(TARGET)" + @mkdir -p "$(OUTPUTDIR)" diff --git a/loader/link.ld b/loader/link.ld new file mode 100644 index 0000000..b275cf2 --- /dev/null +++ b/loader/link.ld @@ -0,0 +1,22 @@ +ENTRY(_start) + +SECTIONS { + PROVIDE(__ipl_start = LDR_LOAD_ADDR); + . = __ipl_start; + .text : { + *(.text._start); + *(._boot_cfg); + *(._ipl_version); + *(._octopus); + *(.text*); + } + .data : { + *(.data*); + *(.rodata*); + *(._payload_00); + *(._payload_01); + } + __ldr_end = .; + . = ALIGN(0x10); + __ipl_end = .; +} diff --git a/loader/loader.c b/loader/loader.c new file mode 100644 index 0000000..2047cb5 --- /dev/null +++ b/loader/loader.c @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2019 CTCaer + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include +#include + +#include "payload_00.h" +#include "payload_01.h" + +#include +#include +#include +#include + +// 0x4003D000: Safe for panic preserving, 0x40038000: Safe for debugging needs. +#define IPL_RELOC_TOP 0x40038000 +#define IPL_PATCHED_RELOC_SZ 0x94 + +boot_cfg_t __attribute__((section ("._boot_cfg"))) b_cfg; +const volatile ipl_ver_meta_t __attribute__((section ("._ipl_version"))) ipl_ver = { + .magic = BL_MAGIC, + .version = (BL_VER_MJ + '0') | ((BL_VER_MN + '0') << 8) | ((BL_VER_HF + '0') << 16), + .rsvd0 = 0, + .rsvd1 = 0 +}; + +const volatile char __attribute__((section ("._octopus"))) octopus[] = + "\n" + " ___\n" + " .-' `'.\n" + " / \\\n" + " | ;\n" + " | | ___.--,\n" + " _.._ |0) = (0) | _.---'`__.-( (_.\n" + " __.--'`_.. '.__.\\ '--. \\_.-' ,.--'` `\"\"`\n" + " ( ,.--'` ',__ /./; ;, '.__.'` __\n" + " _`) ) .---.__.' / | |\\ \\__..--\"\" \"\"\"--.,_\n" + " `---' .'.''-._.-'`_./ /\\ '. \\ _.--''````'''--._`-.__.'\n" + " | | .' _.-' | | \\ \\ '. `----`\n" + " \\ \\/ .' \\ \\ '. '-._)\n" + " \\/ / \\ \\ `=.__`'-.\n" + " / /\\ `) ) / / `\"\".`\\\n" + " , _.-'.'\\ \\ / / ( ( / /\n" + " `--'` ) ) .-'.' '.'. | (\n" + " (/` ( (` ) ) '-; [switchbrew]\n"; + +void loader_main() +{ + // Preserve sections. + __asm__ ("" : : "" (b_cfg)); + __asm__ ("" : : "" (ipl_ver)); + __asm__ ("" : : "" (octopus[0])); + + // Preliminary BPMP clocks init. + CLOCK(CLK_RST_CONTROLLER_CLK_SYSTEM_RATE) = 0x10; // Set HCLK div to 2 and PCLK div to 1. + CLOCK(CLK_RST_CONTROLLER_CLK_SOURCE_SYS) = 0; // Set SCLK div to 1. + CLOCK(CLK_RST_CONTROLLER_SCLK_BURST_POLICY) = 0x20004444; // Set clk source to Run and PLLP_OUT2 (204MHz). + CLOCK(CLK_RST_CONTROLLER_SUPER_SCLK_DIVIDER) = 0x80000000; // Enable SUPER_SDIV to 1. + CLOCK(CLK_RST_CONTROLLER_CLK_SYSTEM_RATE) = 2; // Set HCLK div to 1 and PCLK div to 3. + CLOCK(CLK_RST_CONTROLLER_SCLK_BURST_POLICY) = 0x20003333; // Set SCLK to PLLP_OUT (408MHz). + + // Get Loader and Payload size. + u32 payload_size = sizeof(payload_00) + sizeof(payload_01); + u32 *payload_addr = (u32 *)payload_00; + + // Relocate payload to a safer place. + u32 bytes = ALIGN(payload_size, 4) >> 2; + u32 *addr = payload_addr + bytes - 1; + u32 *dst = (u32 *)(IPL_RELOC_TOP - 4); + while (bytes) + { + *dst = *addr; + dst--; + addr--; + bytes--; + } + + // Uncompress payload parts. + u8 *src_addr = (void *)(IPL_RELOC_TOP - ALIGN(payload_size, 4)); + u32 pos = LZ_Uncompress((const u8 *)src_addr, (u8*)IPL_LOAD_ADDR, sizeof(payload_00)); + src_addr += (u32)payload_01 - (u32)payload_00; + LZ_Uncompress((const u8 *)src_addr, (u8*)IPL_LOAD_ADDR + pos, sizeof(payload_01)); + + // Copy over boot configuration storage in case it was set. + memcpy((u8 *)(IPL_LOAD_ADDR + IPL_PATCHED_RELOC_SZ), &b_cfg, sizeof(boot_cfg_t)); + + // Chainload. + void (*ipl_ptr)() = (void *)IPL_LOAD_ADDR; + (*ipl_ptr)(); + + // Halt if we managed to get out of execution. + while (true) + ; +} diff --git a/loader/start.S b/loader/start.S new file mode 100644 index 0000000..1d7a661 --- /dev/null +++ b/loader/start.S @@ -0,0 +1,73 @@ +/* +* Copyright (c) 2018 naehrwert +* +* This program is free software; you can redistribute it and/or modify it +* under the terms and conditions of the GNU General Public License, +* version 2, as published by the Free Software Foundation. +* +* This program is distributed in the hope it will be useful, but WITHOUT +* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +* more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see . +*/ + +.section .text._start +.arm + +.extern _reloc_ipl +.type _reloc_ipl, %function + +.extern memset +.type memset, %function + +.extern loader_main +.type loader_main, %function + +.globl _start +.type _start, %function +_start: + ADR R0, _start + LDR R1, =__ipl_start + CMP R0, R1 + BEQ _real_start + + /* If we are not in the right location already, copy a relocator to upper IRAM. */ + ADR R2, _reloc_ipl + LDR R3, =0x4003FF00 + MOV R4, #(_real_start - _reloc_ipl) +_copy_loop: + LDMIA R2!, {R5} + STMIA R3!, {R5} + SUBS R4, #4 + BNE _copy_loop + + /* Use the relocator to copy ourselves into the right place. */ + LDR R2, =__ipl_end + SUB R2, R2, R1 + LDR R3, =_real_start + LDR R4, =0x4003FF00 + BX R4 + +_reloc_ipl: + LDMIA R0!, {R4-R7} + STMIA R1!, {R4-R7} + SUBS R2, #0x10 + BNE _reloc_ipl + /* Jump to the relocated entry. */ + BX R3 + +_real_start: + /* Initially, we place our stack in IRAM but will move it to SDRAM later. */ + LDR SP, =0x40007000 + LDR R0, =__ldr_end + BL loader_main + B . + .word 0 + .word 0 + .word 0 + .word 0 + .word 0 + .word 0 diff --git a/tools/bin2c/Makefile b/tools/bin2c/Makefile new file mode 100644 index 0000000..303f322 --- /dev/null +++ b/tools/bin2c/Makefile @@ -0,0 +1,12 @@ +NATIVE_CC ?= gcc + +.PHONY: all clean + +all: bin2c + @echo > /dev/null + +clean: + rm -f bin2c + +bin2c: bin2c.c + @$(NATIVE_CC) -o $@ bin2c.c diff --git a/tools/bin2c/bin2c.c b/tools/bin2c/bin2c.c new file mode 100644 index 0000000..33f2687 --- /dev/null +++ b/tools/bin2c/bin2c.c @@ -0,0 +1,77 @@ +/* + * This is bin2c program, which allows you to convert binary file to + * C language array, for use as embedded resource, for instance you can + * embed graphics or audio file directly into your program. + * This is public domain software, use it on your own risk. + * Contact Serge Fukanchik at fuxx@mail.ru if you have any questions. + */ + +#include +#include +#include +#include +#include +#include + +/* Replace . with _ */ +char* +make_ident ( char* name ) +{ + char* ret; + char* p; + + ret = strdup ( name ); + + for ( p = ret; p[0]; p++ ) + { + if ( !isalnum ( p[0] ) ) p[0] = '_'; + } + return ret; +} + +int +main ( int argc, char* argv[] ) +{ + unsigned char buf[BUFSIZ]; + char* ident; + int fd, i, total, rd, need_comma; + + if ( argc < 2 ) + { + fprintf ( stderr, "Usage: %s binary_file > output_file\n", argv[0] ); + return -1; + } + + fd = open ( argv[1], O_RDONLY ); + if ( fd == -1 ) + { + fprintf ( stderr, "%s: can't open %s for reading\n", argv[0], argv[1] ); + return -1; + } + + ident = make_ident ( argv[1] ); + + printf ( "static const unsigned char __attribute__((section (\"._%s\"))) %s[] = {", ident, ident ); + for ( total = 0, need_comma = 0; ( rd = read ( fd, buf, BUFSIZ ) ) != 0; ) + { + if ( rd == -1 ) + { + fprintf ( stderr, "%s: file read error\n", argv[0] ); + return -1; + } + for ( i = 0; i < rd; i++ ) + { + if ( need_comma ) printf ( ", " ); + else need_comma = 1; + if ( ( total % 11 ) == 0 ) printf ( "\n\t" ); + printf ( "0x%.2x", buf[i] ); + total++; + } + } + printf ( "\n};\n" ); + + close ( fd ); + free ( ident ); + + return 0; +} diff --git a/tools/lz/Makefile b/tools/lz/Makefile new file mode 100644 index 0000000..8ffabde --- /dev/null +++ b/tools/lz/Makefile @@ -0,0 +1,12 @@ +NATIVE_CC ?= gcc + +.PHONY: all clean + +all: lz77 + @echo > /dev/null + +clean: + rm -f lz77 + +lz77: lz.c lz77.c + @$(NATIVE_CC) -o $@ lz.c lz77.c diff --git a/tools/lz/lz.c b/tools/lz/lz.c new file mode 100644 index 0000000..01a3e69 --- /dev/null +++ b/tools/lz/lz.c @@ -0,0 +1,546 @@ +// +// Name: lz.c +// Author: Marcus Geelnard +// Description: LZ77 coder/decoder implementation. +// Reentrant: Yes +// $ATH_LICENSE_NULL$ +// +// The LZ77 compression scheme is a substitutional compression scheme +// proposed by Abraham Lempel and Jakob Ziv in 1977. It is very simple in +// its design, and uses no fancy bit level compression. +// +// This is my first attempt at an implementation of a LZ77 code/decoder. +// +// The principle of the LZ77 compression algorithm is to store repeated +// occurrences of strings as references to previous occurrences of the same +// string. The point is that the reference consumes less space than the +// string itself, provided that the string is long enough (in this +// implementation, the string has to be at least 4 bytes long, since the +// minimum coded reference is 3 bytes long). Also note that the term +// "string" refers to any kind of byte sequence (it does not have to be +// an ASCII string, for instance). +// +// The coder uses a brute force approach to finding string matches in the +// history buffer (or "sliding window", if you wish), which is very, very +// slow. I recon the complexity is somewhere between O(n^2) and O(n^3), +// depending on the input data. +// +// There is also a faster implementation that uses a large working buffer +// in which a "jump table" is stored, which is used to quickly find +// possible string matches (see the source code for LZ_CompressFast() for +// more information). The faster method is an order of magnitude faster, +// but still quite slow compared to other compression methods. +// +// The upside is that decompression is very fast, and the compression ratio +// is often very good. +// +// The reference to a string is coded as a (length,offset) pair, where the +// length indicates the length of the string, and the offset gives the +// offset from the current data position. To distinguish between string +// references and literal strings (uncompressed bytes), a string reference +// is preceded by a marker byte, which is chosen as the least common byte +// symbol in the input data stream (this marker byte is stored in the +// output stream as the first byte). +// +// Occurrences of the marker byte in the stream are encoded as the marker +// byte followed by a zero byte, which means that occurrences of the marker +// byte have to be coded with two bytes. +// +// The lengths and offsets are coded in a variable length fashion, allowing +// values of any magnitude (up to 4294967295 in this implementation). +// +// With this compression scheme, the worst case compression result is +// (257/256)*insize + 1. +// +//------------------------------------------------------------------------ +// Copyright (c) 2003-2006 Marcus Geelnard +// +// This software is provided 'as-is', without any express or implied +// warranty. In no event will the authors be held liable for any damages +// arising from the use of this software. +// +// Permission is granted to anyone to use this software for any purpose, +// including commercial applications, and to alter it and redistribute it +// freely, subject to the following restrictions: +// +// 1. The origin of this software must not be misrepresented; you must not +// claim that you wrote the original software. If you use this software +// in a product, an acknowledgment in the product documentation would +// be appreciated but is not required. +// +// 2. Altered source versions must be plainly marked as such, and must not +// be misrepresented as being the original software. +// +// 3. This notice may not be removed or altered from any source +// distribution. +// +// Marcus Geelnard +// marcus.geelnard at home.se +// + +// +// This file has been altered from the original version. +// + +/************************************************************************* +* Constants used for LZ77 coding +*************************************************************************/ + +/* Maximum offset (can be any size < 2^31). Lower values give faster + compression, while higher values gives better compression. The default + value of 100000 is quite high. Experiment to see what works best for + you. */ +#define LZ_MAX_OFFSET 100000 + + + +/************************************************************************* +* INTERNAL FUNCTIONS * +*************************************************************************/ + + +/************************************************************************* +* _LZ_StringCompare() - Return maximum length string match. +*************************************************************************/ + +static unsigned int _LZ_StringCompare( unsigned char * str1, + unsigned char * str2, unsigned int minlen, unsigned int maxlen ) +{ + unsigned int len; + + for( len = minlen; (len < maxlen) && (str1[len] == str2[len]); ++ len ); + + return len; +} + + +/************************************************************************* +* _LZ_WriteVarSize() - Write unsigned integer with variable number of +* bytes depending on value. +*************************************************************************/ + +static int _LZ_WriteVarSize( unsigned int x, unsigned char * buf ) +{ + unsigned int y; + int num_bytes, i, b; + + /* Determine number of bytes needed to store the number x */ + y = x >> 3; + for( num_bytes = 5; num_bytes >= 2; -- num_bytes ) + { + if( y & 0xfe000000 ) break; + y <<= 7; + } + + /* Write all bytes, seven bits in each, with 8:th bit set for all */ + /* but the last byte. */ + for( i = num_bytes-1; i >= 0; -- i ) + { + b = (x >> (i*7)) & 0x0000007f; + if( i > 0 ) + { + b |= 0x00000080; + } + *buf ++ = (unsigned char) b; + } + + /* Return number of bytes written */ + return num_bytes; +} + + +/************************************************************************* +* _LZ_ReadVarSize() - Read unsigned integer with variable number of +* bytes depending on value. +*************************************************************************/ + +static int _LZ_ReadVarSize( unsigned int * x, unsigned char * buf ) +{ + unsigned int y, b, num_bytes; + + /* Read complete value (stop when byte contains zero in 8:th bit) */ + y = 0; + num_bytes = 0; + do + { + b = (unsigned int) (*buf ++); + y = (y << 7) | (b & 0x0000007f); + ++ num_bytes; + } + while( b & 0x00000080 ); + + /* Store value in x */ + *x = y; + + /* Return number of bytes read */ + return num_bytes; +} + + + +/************************************************************************* +* PUBLIC FUNCTIONS * +*************************************************************************/ + + +/************************************************************************* +* LZ_Compress() - Compress a block of data using an LZ77 coder. +* in - Input (uncompressed) buffer. +* out - Output (compressed) buffer. This buffer must be 0.4% larger +* than the input buffer, plus one byte. +* insize - Number of input bytes. +* The function returns the size of the compressed data. +*************************************************************************/ + +int LZ_Compress( unsigned char *in, unsigned char *out, + unsigned int insize ) +{ + unsigned char marker, symbol; + unsigned int inpos, outpos, bytesleft, i; + unsigned int maxoffset, offset, bestoffset; + unsigned int maxlength, length, bestlength; + unsigned int histogram[ 256 ]; + unsigned char *ptr1, *ptr2; + + /* Do we have anything to compress? */ + if( insize < 1 ) + { + return 0; + } + + /* Create histogram */ + for( i = 0; i < 256; ++ i ) + { + histogram[ i ] = 0; + } + for( i = 0; i < insize; ++ i ) + { + ++ histogram[ in[ i ] ]; + } + + /* Find the least common byte, and use it as the marker symbol */ + marker = 0; + for( i = 1; i < 256; ++ i ) + { + if( histogram[ i ] < histogram[ marker ] ) + { + marker = i; + } + } + + /* Remember the marker symbol for the decoder */ + out[ 0 ] = marker; + + /* Start of compression */ + inpos = 0; + outpos = 1; + + /* Main compression loop */ + bytesleft = insize; + do + { + /* Determine most distant position */ + if( inpos > LZ_MAX_OFFSET ) maxoffset = LZ_MAX_OFFSET; + else maxoffset = inpos; + + /* Get pointer to current position */ + ptr1 = &in[ inpos ]; + + /* Search history window for maximum length string match */ + bestlength = 3; + bestoffset = 0; + for( offset = 3; offset <= maxoffset; ++ offset ) + { + /* Get pointer to candidate string */ + ptr2 = &ptr1[ -(int)offset ]; + + /* Quickly determine if this is a candidate (for speed) */ + if( (ptr1[ 0 ] == ptr2[ 0 ]) && + (ptr1[ bestlength ] == ptr2[ bestlength ]) ) + { + /* Determine maximum length for this offset */ + maxlength = (bytesleft < offset ? bytesleft : offset); + + /* Count maximum length match at this offset */ + length = _LZ_StringCompare( ptr1, ptr2, 0, maxlength ); + + /* Better match than any previous match? */ + if( length > bestlength ) + { + bestlength = length; + bestoffset = offset; + } + } + } + + /* Was there a good enough match? */ + if( (bestlength >= 8) || + ((bestlength == 4) && (bestoffset <= 0x0000007f)) || + ((bestlength == 5) && (bestoffset <= 0x00003fff)) || + ((bestlength == 6) && (bestoffset <= 0x001fffff)) || + ((bestlength == 7) && (bestoffset <= 0x0fffffff)) ) + { + out[ outpos ++ ] = (unsigned char) marker; + outpos += _LZ_WriteVarSize( bestlength, &out[ outpos ] ); + outpos += _LZ_WriteVarSize( bestoffset, &out[ outpos ] ); + inpos += bestlength; + bytesleft -= bestlength; + } + else + { + /* Output single byte (or two bytes if marker byte) */ + symbol = in[ inpos ++ ]; + out[ outpos ++ ] = symbol; + if( symbol == marker ) + { + out[ outpos ++ ] = 0; + } + -- bytesleft; + } + } + while( bytesleft > 3 ); + + /* Dump remaining bytes, if any */ + while( inpos < insize ) + { + if( in[ inpos ] == marker ) + { + out[ outpos ++ ] = marker; + out[ outpos ++ ] = 0; + } + else + { + out[ outpos ++ ] = in[ inpos ]; + } + ++ inpos; + } + + return outpos; +} + + +/************************************************************************* +* LZ_CompressFast() - Compress a block of data using an LZ77 coder. +* in - Input (uncompressed) buffer. +* out - Output (compressed) buffer. This buffer must be 0.4% larger +* than the input buffer, plus one byte. +* insize - Number of input bytes. +* work - Pointer to a temporary buffer (internal working buffer), which +* must be able to hold (insize+65536) unsigned integers. +* The function returns the size of the compressed data. +*************************************************************************/ + +int LZ_CompressFast( unsigned char *in, unsigned char *out, + unsigned int insize, unsigned int *work ) +{ + unsigned char marker, symbol; + unsigned int inpos, outpos, bytesleft, i, index, symbols; + unsigned int offset, bestoffset; + unsigned int maxlength, length, bestlength; + unsigned int histogram[ 256 ], *lastindex, *jumptable; + unsigned char *ptr1, *ptr2; + + /* Do we have anything to compress? */ + if( insize < 1 ) + { + return 0; + } + + /* Assign arrays to the working area */ + lastindex = work; + jumptable = &work[ 65536 ]; + + /* Build a "jump table". Here is how the jump table works: + jumptable[i] points to the nearest previous occurrence of the same + symbol pair as in[i]:in[i+1], so in[i] == in[jumptable[i]] and + in[i+1] == in[jumptable[i]+1], and so on... Following the jump table + gives a dramatic boost for the string search'n'match loop compared + to doing a brute force search. The jump table is built in O(n) time, + so it is a cheap operation in terms of time, but it is expensice in + terms of memory consumption. */ + for( i = 0; i < 65536; ++ i ) + { + lastindex[ i ] = 0xffffffff; + } + for( i = 0; i < insize-1; ++ i ) + { + symbols = (((unsigned int)in[i]) << 8) | ((unsigned int)in[i+1]); + index = lastindex[ symbols ]; + lastindex[ symbols ] = i; + jumptable[ i ] = index; + } + jumptable[ insize-1 ] = 0xffffffff; + + /* Create histogram */ + for( i = 0; i < 256; ++ i ) + { + histogram[ i ] = 0; + } + for( i = 0; i < insize; ++ i ) + { + ++ histogram[ in[ i ] ]; + } + + /* Find the least common byte, and use it as the marker symbol */ + marker = 0; + for( i = 1; i < 256; ++ i ) + { + if( histogram[ i ] < histogram[ marker ] ) + { + marker = i; + } + } + + /* Remember the marker symbol for the decoder */ + out[ 0 ] = marker; + + /* Start of compression */ + inpos = 0; + outpos = 1; + + /* Main compression loop */ + bytesleft = insize; + do + { + /* Get pointer to current position */ + ptr1 = &in[ inpos ]; + + /* Search history window for maximum length string match */ + bestlength = 3; + bestoffset = 0; + index = jumptable[ inpos ]; + while( (index != 0xffffffff) && ((inpos - index) < LZ_MAX_OFFSET) ) + { + /* Get pointer to candidate string */ + ptr2 = &in[ index ]; + + /* Quickly determine if this is a candidate (for speed) */ + if( ptr2[ bestlength ] == ptr1[ bestlength ] ) + { + /* Determine maximum length for this offset */ + offset = inpos - index; + maxlength = (bytesleft < offset ? bytesleft : offset); + + /* Count maximum length match at this offset */ + length = _LZ_StringCompare( ptr1, ptr2, 2, maxlength ); + + /* Better match than any previous match? */ + if( length > bestlength ) + { + bestlength = length; + bestoffset = offset; + } + } + + /* Get next possible index from jump table */ + index = jumptable[ index ]; + } + + /* Was there a good enough match? */ + if( (bestlength >= 8) || + ((bestlength == 4) && (bestoffset <= 0x0000007f)) || + ((bestlength == 5) && (bestoffset <= 0x00003fff)) || + ((bestlength == 6) && (bestoffset <= 0x001fffff)) || + ((bestlength == 7) && (bestoffset <= 0x0fffffff)) ) + { + out[ outpos ++ ] = (unsigned char) marker; + outpos += _LZ_WriteVarSize( bestlength, &out[ outpos ] ); + outpos += _LZ_WriteVarSize( bestoffset, &out[ outpos ] ); + inpos += bestlength; + bytesleft -= bestlength; + } + else + { + /* Output single byte (or two bytes if marker byte) */ + symbol = in[ inpos ++ ]; + out[ outpos ++ ] = symbol; + if( symbol == marker ) + { + out[ outpos ++ ] = 0; + } + -- bytesleft; + } + } + while( bytesleft > 3 ); + + /* Dump remaining bytes, if any */ + while( inpos < insize ) + { + if( in[ inpos ] == marker ) + { + out[ outpos ++ ] = marker; + out[ outpos ++ ] = 0; + } + else + { + out[ outpos ++ ] = in[ inpos ]; + } + ++ inpos; + } + + return outpos; +} + + +/************************************************************************* +* LZ_Uncompress() - Uncompress a block of data using an LZ77 decoder. +* in - Input (compressed) buffer. +* out - Output (uncompressed) buffer. This buffer must be large +* enough to hold the uncompressed data. +* insize - Number of input bytes. +*************************************************************************/ + +int LZ_Uncompress( unsigned char *in, unsigned char *out, + unsigned int insize ) +{ + unsigned char marker, symbol; + unsigned int i, inpos, outpos, length, offset; + + /* Do we have anything to uncompress? */ + if( insize < 1 ) + { + return 0; + } + + /* Get marker symbol from input stream */ + marker = in[ 0 ]; + inpos = 1; + + /* Main decompression loop */ + outpos = 0; + do + { + symbol = in[ inpos ++ ]; + if( symbol == marker ) + { + /* We had a marker byte */ + if( in[ inpos ] == 0 ) + { + /* It was a single occurrence of the marker byte */ + out[ outpos ++ ] = marker; + ++ inpos; + } + else + { + /* Extract true length and offset */ + inpos += _LZ_ReadVarSize( &length, &in[ inpos ] ); + inpos += _LZ_ReadVarSize( &offset, &in[ inpos ] ); + + /* Copy corresponding data from history window */ + for( i = 0; i < length; ++ i ) + { + out[ outpos ] = out[ outpos - offset ]; + ++ outpos; + } + } + } + else + { + /* No marker, plain copy */ + out[ outpos ++ ] = symbol; + } + } + while( inpos < insize ); + + return outpos; +} diff --git a/tools/lz/lz.h b/tools/lz/lz.h new file mode 100644 index 0000000..7f7b22a --- /dev/null +++ b/tools/lz/lz.h @@ -0,0 +1,61 @@ +// +// Name: lz.h +// Author: Marcus Geelnard +// Description: LZ77 coder/decoder interface. +// Reentrant: Yes +// ------------------------------------------------------------------------ +// $ATH_LICENSE_NULL$ +// Copyright (c) 2003-2006 Marcus Geelnard +// +// This software is provided 'as-is', without any express or implied +// warranty. In no event will the authors be held liable for any damages +// arising from the use of this software. +// +// Permission is granted to anyone to use this software for any purpose, +// including commercial applications, and to alter it and redistribute it +// freely, subject to the following restrictions: +// +// 1. The origin of this software must not be misrepresented; you must not +// claim that you wrote the original software. If you use this software +// in a product, an acknowledgment in the product documentation would +// be appreciated but is not required. +// +// 2. Altered source versions must be plainly marked as such, and must not +// be misrepresented as being the original software. +// +// 3. This notice may not be removed or altered from any source +// distribution. +// +// Marcus Geelnard +// marcus.geelnard at home.se +// + +// +// This file has been altered from the original version. +// + +#ifndef _lz_h_ +#define _lz_h_ + +#ifdef __cplusplus +extern "C" { +#endif + + +/************************************************************************* +* Function prototypes +*************************************************************************/ + +int LZ_Compress( unsigned char *in, unsigned char *out, + unsigned int insize ); +int LZ_CompressFast( unsigned char *in, unsigned char *out, + unsigned int insize, unsigned int *work ); +int LZ_Uncompress( unsigned char *in, unsigned char *out, + unsigned int insize ); + + +#ifdef __cplusplus +} +#endif + +#endif /* _lz_h_ */ diff --git a/tools/lz/lz77.c b/tools/lz/lz77.c new file mode 100644 index 0000000..812ae8d --- /dev/null +++ b/tools/lz/lz77.c @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2019 CTCaer + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include +#include +#include "lz.h" + +char filename[1024]; + +int main(int argc, char *argv[]) +{ + int nbytes; + int filename_len; + struct stat statbuf; + FILE *in_file, *out_file; + + if(stat(argv[1], &statbuf)) + goto error; + + if((in_file=fopen(argv[1], "r")) == NULL) + goto error; + + strcpy(filename, argv[1]); + filename_len = strlen(filename); + + uint32_t in_size = statbuf.st_size; + uint8_t *in_buf = (uint8_t *)malloc(in_size); + + uint32_t out_size = statbuf.st_size + 257; + uint8_t *out_buf = (uint8_t *)malloc(out_size); + + if(!(in_buf && out_buf)) + goto error; + + if(fread(in_buf, 1, in_size, in_file) != in_size) + goto error; + + fclose(in_file); + + uint32_t *work = (uint32_t*)malloc(sizeof(uint32_t) * (in_size + 65536)); + for (int i = 0; i < 2; i++) + { + uint32_t in_size_tmp; + if (!i) + { + in_size_tmp = in_size / 2; + strcpy(filename + filename_len, ".00.lz"); + } + else + { + in_size_tmp = in_size - (in_size / 2); + strcpy(filename + filename_len, ".01.lz"); + } + + if (work) + nbytes = LZ_CompressFast(in_buf + (in_size / 2) * i, out_buf, in_size_tmp, work); + else + goto error; + + if (nbytes > out_size) + goto error; + + if((out_file = fopen(filename,"w")) == NULL) + goto error; + + if (fwrite(out_buf, 1, nbytes, out_file) != nbytes) + goto error; + + fclose(out_file); + } + + return 0; + +error: + fprintf(stderr, "Failed to compress: %s\n", argv[1]); + exit(1); +}